From 238d770d5ecc3576d2489e7107aab443ee83b12f Mon Sep 17 00:00:00 2001 From: booboy Date: Thu, 7 Feb 2019 01:10:20 -0600 Subject: [PATCH] this is now namedspaced correctly. at least much better than it was before this --- README.md | 2 +- bin/console | 17 +- bin/pastebinner | 13 +- examples/examples.rb | 14 +- helpers/elastic_search_helper.rb | 47 ----- helpers/sidekiq_helper.rb | 12 -- lib/exceptions.rb | 13 -- lib/pastebinner.rb | 194 +----------------- lib/pastebinner/api_client.rb | 184 +++++++++++++++++ lib/pastebinner/exceptions.rb | 5 - .../helpers/elastic_search_helper.rb | 49 +++++ lib/pastebinner/option_parser.rb | 74 +++---- lib/pastebinner/workers/paste_to_es.rb | 17 ++ workers/paste_to_es.rb | 15 -- 14 files changed, 315 insertions(+), 341 deletions(-) mode change 100755 => 100644 examples/examples.rb delete mode 100644 helpers/elastic_search_helper.rb delete mode 100644 helpers/sidekiq_helper.rb delete mode 100644 lib/exceptions.rb create mode 100644 lib/pastebinner/api_client.rb delete mode 100644 lib/pastebinner/exceptions.rb create mode 100644 lib/pastebinner/helpers/elastic_search_helper.rb create mode 100644 lib/pastebinner/workers/paste_to_es.rb delete mode 100644 workers/paste_to_es.rb diff --git a/README.md b/README.md index 3f407ac..b560cf1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ pastebin_password ``` If you want to scrape pastes, you can view a json response of the latest pastes by using the ```-s``` or ```--scrape_public``` options. -Creating pastes is built in, check ```lib/pastebinner.rb```, I am still working on adding in the functionality to use it on the command line. +Creating pastes is built in, check ```lib/api_client.rb```, I am still working on adding in the functionality to use it on the command line. The command line app can be used as follows: ```shell diff --git a/bin/console b/bin/console index 745bb7a..069d4bd 100755 --- a/bin/console +++ b/bin/console @@ -2,21 +2,12 @@ require 'sidekiq' require 'sidekiq/api' -require 'pry' - -require '../lib/pastebinner' -require '../helpers/elastic_search_helper' -require '../workers/paste_to_es' -require '../config/initializers/sidekiq' +require 'bundler/setup' +require 'pastebinner' require 'pry' - -# set restclient logging and setup a pb object -RestClient.log ='stdout' - -# setup a pastebinner object and elastic search objects so we have them to work with right when we launch console -pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) -es = ElasticSearchHelper.new(ENV['elastic_search_url'], 'pastes') +pb = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) +es = Pastebinner::ElasticSearchHelper.new(ENV['elastic_search_url'], 'pastes') binding.pry diff --git a/bin/pastebinner b/bin/pastebinner index bf85048..10f6399 100755 --- a/bin/pastebinner +++ b/bin/pastebinner @@ -1,10 +1,11 @@ #!/usr/bin/env ruby -require '../lib/pastebinner' -require '../lib/pastebinner/option_parser' -require 'pry' +require 'sidekiq' +require 'sidekiq/api' +require 'bundler/setup' +require 'pastebinner' # setup our object and grab a session key -pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) +pb = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) # set the commandline client to grab 50 pastes by default. this should be an option to config though once we add configuration methods paste_max = 50 @@ -39,10 +40,10 @@ def download_pastes_raw(pb, paste_max) end end -options = OptionParser.parse! +options = Pastebinner::OptionParser.parse! if options[:s] - puts pb.scrape_public_paste(paste_max) + puts pb.scrape_public_pastes(paste_max) elsif options[:r] && options[:k] key = options[:k] puts pb.raw_paste_data(key) diff --git a/examples/examples.rb b/examples/examples.rb old mode 100755 new mode 100644 index 706c917..e97e8e4 --- a/examples/examples.rb +++ b/examples/examples.rb @@ -1,14 +1,7 @@ -#!/usr/bin/env ruby - -require '../lib/pastebinner' - -######################## TESTING #################################################### -##################################################################################### - #### INITIAL STEPS - +# # setup our object and grab a session key -pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) +pb = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) api_dev_key = ENV['pastebin_api_key'] #### CREATE PASTE @@ -29,3 +22,6 @@ puts pb.raw_paste_data('Gkb4ukK9') #### SCRAPE RAW METADATA OF A PASTE KEY (WORKS WITH WHITELISTED IP ONLY) puts pb.raw_paste_metadata('Gkb4ukK9') + + +### MORE EXAMPLES TO COME diff --git a/helpers/elastic_search_helper.rb b/helpers/elastic_search_helper.rb deleted file mode 100644 index 68dc6ba..0000000 --- a/helpers/elastic_search_helper.rb +++ /dev/null @@ -1,47 +0,0 @@ -class ElasticSearchHelper - attr_accessor :server_uri, :index, :pastebinner, :doctype - DEFAULT_METHOD = :post - - def initialize(server_uri, index, doctype='paste') - @server_uri = server_uri - @index = index - @doctype = doctype - @pastebinner = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) - end - - def delete_index - response = RestClient::Request.execute( - method: :delete, - url: "#{server_uri}/#{index}") - end - - def get_mappings - response = RestClient::Request.execute( - method: :get, - url: "#{server_uri}/#{index}/_mappings?pretty") - end - - def update_mapping(mapping_json) - header = { 'Content-type': 'application/json' } - response = RestClient::Request.execute( - method: :put, - url: "#{server_uri}/#{index}/_mapping/#{doctype}", - payload: mapping_json, - headers: header) - end - - def json_to_es(paste_json, method=nil) - header = { 'Content-type': 'application/json' } - response = RestClient::Request.execute( - method: method ||= DEFAULT_METHOD, - url: "#{server_uri}/#{index}/#{doctype}", - headers: header, - payload: paste_json) - end - - def json_to_es_bulk(array_of_paste_json) - array_of_paste_json.each do |paste_json| - self.json_to_es(paste_json) - end - end -end diff --git a/helpers/sidekiq_helper.rb b/helpers/sidekiq_helper.rb deleted file mode 100644 index 5865378..0000000 --- a/helpers/sidekiq_helper.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'sidekiq' -require 'sidekiq/api' - -# load up the redis cfg -require '../config/initializers/sidekiq' - -class SidekiqHelper - - def initialize - end - -end diff --git a/lib/exceptions.rb b/lib/exceptions.rb deleted file mode 100644 index 9cb5050..0000000 --- a/lib/exceptions.rb +++ /dev/null @@ -1,13 +0,0 @@ -module PastebinnerError - class ArgumentError < StandardError - def message - 'Invalid argument' - end - end - - class ConfigError < StandardError - def message - 'Invalid configuration' - end - end -end diff --git a/lib/pastebinner.rb b/lib/pastebinner.rb index 904432e..2b0770c 100755 --- a/lib/pastebinner.rb +++ b/lib/pastebinner.rb @@ -1,188 +1,16 @@ -#!/usr/bin/env ruby -# author: brendan mcdevitt -# a ruby wrapper around all of the methods pastebin provides with its api -# official docs from pastebin on their api can be found at https://pastebin.com/api +# change this to a config module soon: +require '../config/initializers/sidekiq' require 'rest-client' require 'json' require 'yaml' +require 'optparse' +require 'pastebinner/version' +require 'pastebinner/option_parser' +require 'pastebinner/api_client' +require 'pastebinner/helpers/elastic_search_helper' +require 'pastebinner/workers/paste_to_es' -class Pastebinner - attr_accessor :api_dev_key, :username, :password - - def initialize(api_dev_key, username, password) - @api_dev_key = api_dev_key - @username = username - @password = password - @base_api_url = 'https://pastebin.com/api' - @scraping_api_url = 'https://scrape.pastebin.com' - end - - # this should be a hash of { endpoint_name: '/url_endpoint.php'} - ENDPOINTS = { login: '/api_login.php', - post: '/api_post.php', - raw: '/api_raw.php', - scraping: '/api_scraping.php', - scrape_item: '/api_scrape_item.php', - scrape_item_meta: '/api_scrape_item_meta.php' }.freeze - - # basic example hash for creating a paste: - # params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data} - - # required params: - # api_dev_key - your unique developer api key - # api_option - set as paste, this will indicate you want to create a new paste - # api_paste_code - this is the text that will be written inside of your paste - - # optional params: - # api_user_key - this parameter is part of the login system, which is explained further down the page - # api_paste_name - this will be the name / title of your paste - # api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page - # api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2 - # api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page - - # example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data } - def create_paste(params) - execute_query(:api_post, params) - end - - def api_user_key - # returns a user session key that can be used as the api_user_key param - @api_user_key ||= RestClient::Request.execute( - method: :post, - url: @base_api_url + ENDPOINTS[:login], - payload: { 'api_dev_key': @api_dev_key, - 'api_user_name': @username, - 'api_user_password': @password } - ) - end - - def list_user_pastes - params = { 'api_dev_key': api_dev_key, - 'api_user_key': api_user_key, - 'api_results_limit': '100', - 'api_option': 'list' } - execute_query(:api_post, params) - end - - def list_trending_pastes - params = { 'api_dev_key': api_dev_key, - 'api_option': 'trends' } - execute_query(:api_post, params) - end - - def list_raw_user_paste(api_paste_key) - params = { 'api_dev_key': api_dev_key, - 'api_user_key': api_user_key, - 'api_paste_key': api_paste_key, - 'api_option': 'show_paste' } - execute_query(:api_post, params) - end - - # api_paste_key = this is the unique key of the paste data you want to delete. - def delete_user_paste(api_paste_key) - params = { 'api_dev_key': api_dev_key, - 'api_user_key': api_user_key, - 'api_paste_key': api_paste_key, - 'api_option': 'delete' } - execute_query(:api_post, params) - end - - def get_user_info - params = { 'api_dev_key': api_dev_key } - end - - def api_post(params) - response = RestClient::Request.execute( - method: :post, - url: @base_api_url + ENDPOINTS[:post], - payload: params - ) - end - - # params is optional for now. to query specific language ?lang=ruby as an example - # right now its set to grab the max 250, default is 50. param is ?limit=value - def scrape_public_pastes(_params = nil, limit) - response = RestClient::Request.execute( - method: :get, - url: @scraping_api_url + ENDPOINTS[:scraping] + "?limit=#{limit}" - ) - end - - # will extract just the keys from recent public pastes - def get_unique_paste_keys(public_pastes) - pp = JSON.parse(public_pastes) - pp.map { |p| p['key'] } - end - - # scraped keys difference returned - # https://stackoverflow.com/questions/8639857/rails-3-how-to-get-the-difference-between-two-arrays - # note ruby 2.6 has a method for this - https://github.com/ruby/ruby/blob/trunk/array.c#L4450-L4563 - def difference(a, b) - a - b | b - a - end - - def raw_paste_data(unique_paste_key) - response = RestClient::Request.execute( - method: :get, - url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}" - ) - end - - def raw_paste_metadata(unique_paste_key) - response = RestClient::Request.execute( - method: :get, - url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}" - ) - YAML.safe_load(response.body).first - end - - def hash_pastes(keys) - if keys.is_a? String - raw_paste = self.raw_paste_data(keys) - raw_paste_metadata = self.raw_paste_metadata(keys) - hash = self.hash_paste(raw_paste, raw_paste_metadata) - else - keys.map do |key| - raw_paste = self.raw_paste_data(key).body - raw_paste_metadata = self.raw_paste_metadata(key) - hash = self.hash_paste(raw_paste, raw_paste_metadata) - end - end - end - - def hash_paste(raw_paste_data, raw_paste_metadata) - { "paste_metadata": raw_paste_metadata, - "paste_text": raw_paste_data } - end - - def json_paste(key=nil, keys) - # if we give keys, create an array of X json pastes - if keys - self.hash_pastes(keys).map do |paste_hash| - paste_hash.to_json - end - else - # otherwise, just make a json of the 1 raw_paste_data & raw_paste_metadata - raw_paste_data = self.raw_paste_data(key) - raw_paste_metadata = self.raw_paste_metadata(key) - self.hash_paste(raw_paste_data, raw_paste_metadata).to_json - end - end - - def json_paste_from_file(raw_paste_json_file) - raw_paste_json = File.read(raw_paste_json_file) - self.hash_paste(raw_paste_json).to_json - end - - # keep this method private so we are not letting anyone run any method in our program - private - - # this will be the main way to execute any of these methods. this has the exception handling taken care of. - def execute_query(selector, *args) - send(selector, *args) - rescue RestClient::ExceptionWithResponse => e - puts e.message - end - # make my own exception class - # inherit ruby standard error class +module Pastebinner + class Error < StandardError; end + # your code goes here end diff --git a/lib/pastebinner/api_client.rb b/lib/pastebinner/api_client.rb new file mode 100644 index 0000000..3e49957 --- /dev/null +++ b/lib/pastebinner/api_client.rb @@ -0,0 +1,184 @@ +# author: brendan mcdevitt +# a ruby wrapper around all of the methods pastebin provides with its api +# official docs from pastebin on their api can be found at https://pastebin.com/api + +module Pastebinner + class ApiClient + attr_accessor :api_dev_key, :username, :password + + def initialize(api_dev_key, username, password) + @api_dev_key = api_dev_key + @username = username + @password = password + @base_api_url = 'https://pastebin.com/api' + @scraping_api_url = 'https://scrape.pastebin.com' + end + + # this should be a hash of { endpoint_name: '/url_endpoint.php'} + ENDPOINTS = { login: '/api_login.php', + post: '/api_post.php', + raw: '/api_raw.php', + scraping: '/api_scraping.php', + scrape_item: '/api_scrape_item.php', + scrape_item_meta: '/api_scrape_item_meta.php' }.freeze + + # basic example hash for creating a paste: + # params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data} + + # required params: + # api_dev_key - your unique developer api key + # api_option - set as paste, this will indicate you want to create a new paste + # api_paste_code - this is the text that will be written inside of your paste + + # optional params: + # api_user_key - this parameter is part of the login system, which is explained further down the page + # api_paste_name - this will be the name / title of your paste + # api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page + # api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2 + # api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page + + # example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data } + def create_paste(params) + execute_query(:api_post, params) + end + + def api_user_key + # returns a user session key that can be used as the api_user_key param + @api_user_key ||= RestClient::Request.execute( + method: :post, + url: @base_api_url + ENDPOINTS[:login], + payload: { 'api_dev_key': @api_dev_key, + 'api_user_name': @username, + 'api_user_password': @password } + ) + end + + def list_user_pastes + params = { 'api_dev_key': api_dev_key, + 'api_user_key': api_user_key, + 'api_results_limit': '100', + 'api_option': 'list' } + execute_query(:api_post, params) + end + + def list_trending_pastes + params = { 'api_dev_key': api_dev_key, + 'api_option': 'trends' } + execute_query(:api_post, params) + end + + def list_raw_user_paste(api_paste_key) + params = { 'api_dev_key': api_dev_key, + 'api_user_key': api_user_key, + 'api_paste_key': api_paste_key, + 'api_option': 'show_paste' } + execute_query(:api_post, params) + end + + # api_paste_key = this is the unique key of the paste data you want to delete. + def delete_user_paste(api_paste_key) + params = { 'api_dev_key': api_dev_key, + 'api_user_key': api_user_key, + 'api_paste_key': api_paste_key, + 'api_option': 'delete' } + execute_query(:api_post, params) + end + + def get_user_info + params = { 'api_dev_key': api_dev_key } + end + + def api_post(params) + response = RestClient::Request.execute( + method: :post, + url: @base_api_url + ENDPOINTS[:post], + payload: params + ) + end + + # params is optional for now. to query specific language ?lang=ruby as an example + # right now its set to grab the max 250, default is 50. param is ?limit=value + def scrape_public_pastes(_params = nil, limit) + response = RestClient::Request.execute( + method: :get, + url: @scraping_api_url + ENDPOINTS[:scraping] + "?limit=#{limit}" + ) + end + + # will extract just the keys from recent public pastes + def get_unique_paste_keys(public_pastes) + pp = JSON.parse(public_pastes) + pp.map { |p| p['key'] } + end + + # scraped keys difference returned + # https://stackoverflow.com/questions/8639857/rails-3-how-to-get-the-difference-between-two-arrays + # note ruby 2.6 has a method for this - https://github.com/ruby/ruby/blob/trunk/array.c#L4450-L4563 + def difference(a, b) + a - b | b - a + end + + def raw_paste_data(unique_paste_key) + response = RestClient::Request.execute( + method: :get, + url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}" + ) + end + + def raw_paste_metadata(unique_paste_key) + response = RestClient::Request.execute( + method: :get, + url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}" + ) + YAML.safe_load(response.body).first + end + + def hash_pastes(keys) + if keys.is_a? String + raw_paste = self.raw_paste_data(keys) + raw_paste_metadata = self.raw_paste_metadata(keys) + hash = self.hash_paste(raw_paste, raw_paste_metadata) + else + keys.map do |key| + raw_paste = self.raw_paste_data(key).body + raw_paste_metadata = self.raw_paste_metadata(key) + hash = self.hash_paste(raw_paste, raw_paste_metadata) + end + end + end + + def hash_paste(raw_paste_data, raw_paste_metadata) + { "paste_metadata": raw_paste_metadata, + "paste_text": raw_paste_data } + end + + def json_paste(key=nil, keys) + # if we give keys, create an array of X json pastes + if keys + self.hash_pastes(keys).map do |paste_hash| + paste_hash.to_json + end + else + # otherwise, just make a json of the 1 raw_paste_data & raw_paste_metadata + raw_paste_data = self.raw_paste_data(key) + raw_paste_metadata = self.raw_paste_metadata(key) + self.hash_paste(raw_paste_data, raw_paste_metadata).to_json + end + end + + def json_paste_from_file(raw_paste_json_file) + raw_paste_json = File.read(raw_paste_json_file) + self.hash_paste(raw_paste_json).to_json + end + + # keep this method private so we are not letting anyone run any method in our program + private + + # this will be the main way to execute any of these methods. this has the exception handling taken care of. + def execute_query(selector, *args) + send(selector, *args) + rescue RestClient::ExceptionWithResponse => e + puts e.message + end + end +end diff --git a/lib/pastebinner/exceptions.rb b/lib/pastebinner/exceptions.rb deleted file mode 100644 index bdc50bb..0000000 --- a/lib/pastebinner/exceptions.rb +++ /dev/null @@ -1,5 +0,0 @@ -class PastebinnerError < StandardError - def InvalidArgument; end - - def ConfigError; end -end diff --git a/lib/pastebinner/helpers/elastic_search_helper.rb b/lib/pastebinner/helpers/elastic_search_helper.rb new file mode 100644 index 0000000..b7e4f33 --- /dev/null +++ b/lib/pastebinner/helpers/elastic_search_helper.rb @@ -0,0 +1,49 @@ +module Pastebinner + class ElasticSearchHelper + attr_accessor :server_uri, :index, :api_client, :doctype + DEFAULT_METHOD = :post + + def initialize(server_uri, index, doctype='paste') + @server_uri = server_uri + @index = index + @doctype = doctype + @api_client = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) + end + + def delete_index + response = RestClient::Request.execute( + method: :delete, + url: "#{server_uri}/#{index}") + end + + def get_mappings + response = RestClient::Request.execute( + method: :get, + url: "#{server_uri}/#{index}/_mappings?pretty") + end + + def update_mapping(mapping_json) + header = { 'Content-type': 'application/json' } + response = RestClient::Request.execute( + method: :put, + url: "#{server_uri}/#{index}/_mapping/#{doctype}", + payload: mapping_json, + headers: header) + end + + def json_to_es(paste_json, method=nil) + header = { 'Content-type': 'application/json' } + response = RestClient::Request.execute( + method: method ||= DEFAULT_METHOD, + url: "#{server_uri}/#{index}/#{doctype}", + headers: header, + payload: paste_json) + end + + def json_to_es_bulk(array_of_paste_json) + array_of_paste_json.each do |paste_json| + self.json_to_es(paste_json) + end + end + end +end diff --git a/lib/pastebinner/option_parser.rb b/lib/pastebinner/option_parser.rb index 2713b87..ab2b120 100644 --- a/lib/pastebinner/option_parser.rb +++ b/lib/pastebinner/option_parser.rb @@ -1,51 +1,51 @@ -require 'optparse' +module Pastebinner + class OptionParser + def self.parse!(argv = ARGV) + options = {} + OptParse.new do |opts| + opts.default_argv = argv -class OptionParser - def self.parse!(argv = ARGV) - options = {} - OptParse.new do |opts| - opts.default_argv = argv + opts.banner = 'Usage: pastebinner [options]' - opts.banner = 'Usage: pastebinner [options]' - - opts.on('-h', '--help', 'Show this help messae') do - puts opts - exit - end + opts.on('-h', '--help', 'Show this help messae') do + puts opts + exit + end - opts.on('-v', '--verbose', 'Verbose http output (WIP)') do |_v| - options[:v] = true - end + opts.on('-v', '--verbose', 'Verbose http output (WIP)') do |_v| + options[:v] = true + end - opts.on('-s', '--scrape_public', 'Scrape public pastes') do |_s| - options[:s] = true - end + opts.on('-s', '--scrape_public', 'Scrape public pastes') do |_s| + options[:s] = true + end - opts.on('-r', '--raw', 'Raw paste. Requires --key passed with a valid key') do |_r| - options[:r] = true - end + opts.on('-r', '--raw', 'Raw paste. Requires --key passed with a valid key') do |_r| + options[:r] = true + end - opts.on('-g', '--get_keys', 'Get unique paste keys from public pastes') do |_g| - options[:g] = true - end + opts.on('-g', '--get_keys', 'Get unique paste keys from public pastes') do |_g| + options[:g] = true + end - opts.on('-k', '--key=', 'Unique paste key') do |k| - options[:k] = k - end + opts.on('-k', '--key=', 'Unique paste key') do |k| + options[:k] = k + end - opts.on('-d', '--download', 'Download all public pastes to data directory') do |_d| - options[:d] = true - end + opts.on('-d', '--download', 'Download all public pastes to data directory') do |_d| + options[:d] = true + end - opts.on('-j', '--json', 'Download all public pastes as a json into data directory') do |_j| - options[:j] = true - end + opts.on('-j', '--json', 'Download all public pastes as a json into data directory') do |_j| + options[:j] = true + end - opts.on('-t', '--trending', 'Trending pastes') do |_t| - options[:t] = true + opts.on('-t', '--trending', 'Trending pastes') do |_t| + options[:t] = true + end + opts.parse! end - opts.parse! + options end - options end end diff --git a/lib/pastebinner/workers/paste_to_es.rb b/lib/pastebinner/workers/paste_to_es.rb new file mode 100644 index 0000000..758ad72 --- /dev/null +++ b/lib/pastebinner/workers/paste_to_es.rb @@ -0,0 +1,17 @@ +module Pastebinner + class PasteToEs + include Sidekiq::Worker + + sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures + def perform(es_object, pb_object, paste_max) + Logger.new(STDOUT).info("PasteToEs started") + # get public pastes and their keys + pastes = pb_object.scrape_public_pastes(paste_max) + keys = pb_object.get_unique_paste_keys(pastes) + + # build it into json and send it to elasticsearch + json_data = pb_object.json_paste(keys) + es_object.json_to_es_bulk(json_data) + end + end +end diff --git a/workers/paste_to_es.rb b/workers/paste_to_es.rb deleted file mode 100644 index 94b5daf..0000000 --- a/workers/paste_to_es.rb +++ /dev/null @@ -1,15 +0,0 @@ -class PasteToEs - include Sidekiq::Worker - - sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures - def perform(es_object, pb_object, paste_max) - Logger.new(STDOUT).info("PasteToEs started") - # get public pastes and their keys - pastes = pb_object.scrape_public_pastes(paste_max) - keys = pb_object.get_unique_paste_keys(pastes) - - # build it into json and send it to elasticsearch - json_data = pb_object.json_paste(keys) - es_object.json_to_es_bulk(json_data) - end -end