From 4853f92ecd36ea20174787193eb3a2e9a3e21cfb Mon Sep 17 00:00:00 2001
From: booboy <booboy@booboy.pc>
Date: Mon, 5 Nov 2018 03:15:29 -0600
Subject: [PATCH] wrote a method to download all the raw pastes to a file
 inside of a directory.

added a --download and -d shorthand switch to download the raw paste files from a public scrape.
i need to check to make sure that it is the correct way to do it, and also check to see if a filename exists already.
this will prevent duplicate files from being downloaded.
---
 .gitignore      |  1 +
 bin/pastebinner | 23 +++++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index a4be4c3..3ab247f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/data
 Gemfile.lock
 
 # rspec failure tracking
diff --git a/bin/pastebinner b/bin/pastebinner
index 3b9187f..9f0f1d1 100755
--- a/bin/pastebinner
+++ b/bin/pastebinner
@@ -6,6 +6,16 @@ require 'optparse'
 pb =  Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
 api_dev_key = ENV['pastebin_api_key']
 
+# pass in the Pastebinner.new client.
+# will download all of the raw pastes from the public scrape results into each own file in data dir.
+def download_pastes(pb)
+  pub_pastes = pb.scrape_public_pastes
+  keys = pb.get_unique_paste_keys(pub_pastes)
+  keys.map do |id|
+    File.write("../data/pastebin_paste_key_#{id[:key]}", pb.raw_paste_data(id[:key]))
+  end
+end
+
 opts = {}
 
 OptionParser.new do |parser|
@@ -27,6 +37,9 @@ OptionParser.new do |parser|
   parser.on('-gk', '--get_keys', "Get unique paste keys from public pastes") do |gk|
     opts[:gk] = true
   end
+  parser.on('-d', '--download', 'Download all public pastes to data directory.') do |d|
+    opts[:d] = true
+  end
   parser.on('-t', '--trending', 'Trending pastes.') do |t|
     opts[:t] = true
   end
@@ -43,9 +56,11 @@ elsif opts[:t]
 elsif opts[:gk]
   r = pb.scrape_public_pastes
   puts pb.get_unique_paste_keys(r)
-else
+elsif opts[:d]
+  puts "Downloading paste data into the data directory..."
+  download_pastes(pb)
+  puts "Complete."
+else opts = false
  puts 'please provide arguments'
+ exit
 end
-
-#pp = pb.scrape_public_pastes
-#puts pb.get_unique_paste_keys(pp)