diff --git a/bin/pastebinner b/bin/pastebinner index 8524acd..7a4e1f5 100755 --- a/bin/pastebinner +++ b/bin/pastebinner @@ -13,12 +13,11 @@ def download_pastes(pb) keys = pb.get_unique_paste_keys(pub_pastes) data_dir = '../data/' filename = 'pastebin_paste_key' - binding.pry keys.map do |id| if File.exist?(data_dir + filename + "_#{id}") puts "#{id} already exists on your filesystem, skipping..." else - File.write(data_dir + filename + "_#{id}", pb.raw_paste_data(id)) + File.write(data_dir + filename + "_#{id}", pb.json_paste(pb.raw_paste_data(id), pb.raw_paste_metadata(id))) end end end diff --git a/lib/pastebinner.rb b/lib/pastebinner.rb index 96ee432..a1da763 100755 --- a/lib/pastebinner.rb +++ b/lib/pastebinner.rb @@ -153,33 +153,6 @@ class Pastebinner self.hash_paste(raw_paste_data, raw_paste_metadata).to_json end - def data_mappings - # metadata mappings - # send a PUT - { - "mappings": { - "_doc": { - "properties": { - "type": { "type": 'keyword' }, - "paste_metadata": { "type": 'nested' }, - "properties": [{ - "scrape_url": { "type": 'string' }, - "full_url": { "type": 'string' }, - "date": { "type": 'string' }, - "size": { "type": 'string' }, - "expire": { "type": 'string' }, - "title": { "type": 'string' }, - "syntax": { "type": 'string' }, - "user": { "type": 'string' }, - "hits": { "type": 'string' } - }], - "paste_text": { "type": 'string' } - } - } - } - } - end - # keep this method private so we are not letting anyone run any method in our program private