bpmcdevitt/pastebinner
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

removed old mapping from pastebinner and also adjusted download to include metadata and fulltext of paste in the json

Browse source
This commit is contained in:
booboy 2019-02-02 03:52:56 -06:00
parent bf64a2f60e
commit e580c572f7
2 changed files with 1 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/pastebinner
View file

@ -13,12 +13,11 @@ def download_pastes(pb)
keys = pb.get_unique_paste_keys(pub_pastes)
data_dir = '../data/'
filename = 'pastebin_paste_key'
binding.pry
keys.map do |id|
if File.exist?(data_dir + filename + "_#{id}")
puts "#{id} already exists on your filesystem, skipping..."
else
File.write(data_dir + filename + "_#{id}", pb.raw_paste_data(id))
File.write(data_dir + filename + "_#{id}", pb.json_paste(pb.raw_paste_data(id), pb.raw_paste_metadata(id)))
end
end
end

27
lib/pastebinner.rb
View file

@ -153,33 +153,6 @@ class Pastebinner
self.hash_paste(raw_paste_data, raw_paste_metadata).to_json
end
def data_mappings
# metadata mappings
# send a PUT
{
"mappings": {
"_doc": {
"properties": {
"type": { "type": 'keyword' },
"paste_metadata": { "type": 'nested' },
"properties": [{
"scrape_url": { "type": 'string' },
"full_url": { "type": 'string' },
"date": { "type": 'string' },
"size": { "type": 'string' },
"expire": { "type": 'string' },
"title": { "type": 'string' },
"syntax": { "type": 'string' },
"user": { "type": 'string' },
"hits": { "type": 'string' }
}],
"paste_text": { "type": 'string' }
}
}
}
}
end
# keep this method private so we are not letting anyone run any method in our program
private