From 8d781e2499234c8bebdabe7febf65aa86ecccebf Mon Sep 17 00:00:00 2001
From: booboy <bpmcdevitt@thelinuxspace.com>
Date: Mon, 4 Feb 2019 03:49:06 -0600
Subject: [PATCH] made a job to scrape pastes and send to es, i got a rate
 limit error from pastebin so i need to slow down the calls i think, its 501
 calls to the, raw metadata file, raw data file, and scrape index

---
 lib/paste_to_es.rb | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 lib/paste_to_es.rb

diff --git a/lib/paste_to_es.rb b/lib/paste_to_es.rb
new file mode 100644
index 0000000..9bf6d07
--- /dev/null
+++ b/lib/paste_to_es.rb
@@ -0,0 +1,10 @@
+class PasteToEs
+  include Sidekiq::Worker
+  def perform(es_object, pb_object)
+    Logger.new(STDOUT).info("PasteToEs started")
+    pastes = pb_object.scrape_public_pastes
+    keys = pb_object.get_unique_paste_keys(pastes)
+    json_data = pb_object.json_paste(keys)
+    es_object.json_to_es_bulk(json_data)
+  end
+end