From 30e014a2f5d7c4624281f7af0cca278e070fa723 Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Tue, 29 Oct 2019 21:01:03 -0500 Subject: [PATCH] converting to standalone gem for basic interaction with pastebin api and cmdline client --- bin/console | 12 +- bin/pastebinner | 2 - config/initializers/sidekiq.rb | 9 -- config/sidekiq.service | 52 -------- examples/sinkiq.rb | 61 ---------- lib/pastebinner.rb | 3 - lib/pastebinner/api_client.rb | 14 --- .../helpers/elastic_search_helper.rb | 58 --------- lib/pastebinner/option_parser.rb | 3 - lib/pastebinner/workers/paste_to_es.rb | 17 --- mappings/create_mappings.json | 114 ------------------ mappings/mappings.json | 113 ----------------- pastebinner.gemspec | 5 +- 13 files changed, 3 insertions(+), 460 deletions(-) delete mode 100644 config/initializers/sidekiq.rb delete mode 100644 config/sidekiq.service delete mode 100644 examples/sinkiq.rb delete mode 100644 lib/pastebinner/helpers/elastic_search_helper.rb delete mode 100644 lib/pastebinner/workers/paste_to_es.rb delete mode 100644 mappings/create_mappings.json delete mode 100644 mappings/mappings.json diff --git a/bin/console b/bin/console index eb3beac..5ad1b26 100755 --- a/bin/console +++ b/bin/console @@ -1,12 +1,9 @@ #!/usr/bin/env ruby -require 'sidekiq' -require 'sidekiq/api' - require 'bundler/setup' require 'pastebinner' - require 'pry' + # setup restclient log to the console # make it into a method to toggle on or off def rest_client_toggle_log(default=false) @@ -19,14 +16,9 @@ Pastebinner.configure do |config| config.api_key = ENV['pastebin_api_key'] config.api_username = ENV['pastebin_username'] config.api_password = ENV['pastebin_password'] - config.elasticsearch_url = ENV['elasticsearch_url'] - config.redis_url = ENV['redis_url'] end pb = Pastebinner::ApiClient.new(Pastebinner.configuration.api_key, Pastebinner.configuration.api_username, Pastebinner.configuration.api_password) -es = Pastebinner::ElasticSearchHelper.new(Pastebinner.configuration.elasticsearch_url, 'pastes') - -# sidekiq worker to send pastes to es -paste_to_es_job = Pastebinner::PasteToEs.new +# you should have a pb object to work with now, have fun in pry! binding.pry diff --git a/bin/pastebinner b/bin/pastebinner index 10f6399..c2790dd 100755 --- a/bin/pastebinner +++ b/bin/pastebinner @@ -1,6 +1,4 @@ #!/usr/bin/env ruby -require 'sidekiq' -require 'sidekiq/api' require 'bundler/setup' require 'pastebinner' diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb deleted file mode 100644 index 35d0e5f..0000000 --- a/config/initializers/sidekiq.rb +++ /dev/null @@ -1,9 +0,0 @@ -# server config -Sidekiq.configure_server do |config| - config.redis = { url: ENV["REDIS_SERVER_URL"] } -end - -# client config -Sidekiq.configure_client do |config| - config.redis = { url: ENV["REDIS_SERVER_URL"] } -end diff --git a/config/sidekiq.service b/config/sidekiq.service deleted file mode 100644 index 8e6c925..0000000 --- a/config/sidekiq.service +++ /dev/null @@ -1,52 +0,0 @@ -# -# systemd unit file for CentOS 7, Ubuntu 15.04 -# -# Customize this file based on your bundler location, app directory, etc. -# Put this in /usr/lib/systemd/system (CentOS) or /lib/systemd/system (Ubuntu). -# Run: -# - systemctl enable sidekiq -# - systemctl {start,stop,restart} sidekiq -# -# This file corresponds to a single Sidekiq process. Add multiple copies -# to run multiple processes (sidekiq-1, sidekiq-2, etc). -# -# See Inspeqtor's Systemd wiki page for more detail about Systemd: -# https://github.com/mperham/inspeqtor/wiki/Systemd -# -[Unit] -Description=sidekiq -# start us only once the network and logging subsystems are available, -# consider adding redis-server.service if Redis is local and systemd-managed. -After=syslog.target network.target - -# See these pages for lots of options: -# http://0pointer.de/public/systemd-man/systemd.service.html -# http://0pointer.de/public/systemd-man/systemd.exec.html -[Service] -Type=simple -WorkingDirectory=/opt/myapp/current -# If you use rbenv: -# ExecStart=/bin/bash -lc '/home/deploy/.rbenv/shims/bundle exec sidekiq -e production' -# If you use the system's ruby: -ExecStart=/usr/local/bin/bundle exec sidekiq -e production -User=deploy -Group=deploy -UMask=0002 - -# Greatly reduce Ruby memory fragmentation and heap usage -# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/ -Environment=MALLOC_ARENA_MAX=2 - -# if we crash, restart -RestartSec=1 -Restart=on-failure - -# output goes to /var/log/syslog -StandardOutput=syslog -StandardError=syslog - -# This will default to "bundler" if we don't specify it -SyslogIdentifier=sidekiq - -[Install] -WantedBy=multi-user.target diff --git a/examples/sinkiq.rb b/examples/sinkiq.rb deleted file mode 100644 index 028c577..0000000 --- a/examples/sinkiq.rb +++ /dev/null @@ -1,61 +0,0 @@ -# Make sure you have Sinatra installed, then start sidekiq with -# ./bin/sidekiq -r ./examples/sinkiq.rb -# Simply run Sinatra with -# ruby examples/sinkiq.rb -# and then browse to http://localhost:4567 -# -require 'sinatra' -require 'sidekiq' -require 'redis' -require 'sidekiq/api' - -$redis = Redis.new - -class SinatraWorker - include Sidekiq::Worker - - def perform(msg="lulz you forgot a msg!") - $redis.lpush("sinkiq-example-messages", msg) - end -end - -get '/' do - stats = Sidekiq::Stats.new - @failed = stats.failed - @processed = stats.processed - @messages = $redis.lrange('sinkiq-example-messages', 0, -1) - erb :index -end - -post '/msg' do - SinatraWorker.perform_async params[:msg] - redirect to('/') -end - -__END__ - -@@ layout - - - Sinatra + Sidekiq - - <%= yield %> - - - -@@ index -

Sinatra + Sidekiq Example

-

Failed: <%= @failed %>

-

Processed: <%= @processed %>

- -
- - -
- - Refresh page - -

Messages

- <% @messages.each do |msg| %> -

<%= msg %>

- <% end %> diff --git a/lib/pastebinner.rb b/lib/pastebinner.rb index 7da2b72..4c3fac2 100755 --- a/lib/pastebinner.rb +++ b/lib/pastebinner.rb @@ -1,6 +1,5 @@ # change this to a config module soon: require '../config/configuration' -require '../config/initializers/sidekiq' require 'rest-client' require 'json' require 'yaml' @@ -8,8 +7,6 @@ require 'optparse' require 'pastebinner/version' require 'pastebinner/option_parser' require 'pastebinner/api_client' -require 'pastebinner/helpers/elastic_search_helper' -require 'pastebinner/workers/paste_to_es' module Pastebinner diff --git a/lib/pastebinner/api_client.rb b/lib/pastebinner/api_client.rb index 3e49957..477a155 100644 --- a/lib/pastebinner/api_client.rb +++ b/lib/pastebinner/api_client.rb @@ -61,12 +61,6 @@ module Pastebinner execute_query(:api_post, params) end - def list_trending_pastes - params = { 'api_dev_key': api_dev_key, - 'api_option': 'trends' } - execute_query(:api_post, params) - end - def list_raw_user_paste(api_paste_key) params = { 'api_dev_key': api_dev_key, 'api_user_key': api_user_key, @@ -111,13 +105,6 @@ module Pastebinner pp.map { |p| p['key'] } end - # scraped keys difference returned - # https://stackoverflow.com/questions/8639857/rails-3-how-to-get-the-difference-between-two-arrays - # note ruby 2.6 has a method for this - https://github.com/ruby/ruby/blob/trunk/array.c#L4450-L4563 - def difference(a, b) - a - b | b - a - end - def raw_paste_data(unique_paste_key) response = RestClient::Request.execute( method: :get, @@ -171,7 +158,6 @@ module Pastebinner self.hash_paste(raw_paste_json).to_json end - # keep this method private so we are not letting anyone run any method in our program private # this will be the main way to execute any of these methods. this has the exception handling taken care of. diff --git a/lib/pastebinner/helpers/elastic_search_helper.rb b/lib/pastebinner/helpers/elastic_search_helper.rb deleted file mode 100644 index b41f3fb..0000000 --- a/lib/pastebinner/helpers/elastic_search_helper.rb +++ /dev/null @@ -1,58 +0,0 @@ -module Pastebinner - class ElasticSearchHelper - attr_accessor :server_uri, :index, :api_client, :doctype - DEFAULT_METHOD = :post - - def initialize(server_uri, index, doctype='paste') - @server_uri = server_uri - @index = index - @doctype = doctype - @api_client = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) - end - - def delete_index - response = RestClient::Request.execute( - method: :delete, - url: "#{server_uri}/#{index}") - end - - def get_mappings - response = RestClient::Request.execute( - method: :get, - url: "#{server_uri}/#{index}/_mappings?pretty") - end - - def create_mapping(mapping_json) - header = { 'Content-type': 'application/json' } - response = RestClient::Request.execute( - method: :put, - url: "#{server_uri}/#{index}", - headers: header, - payload: mapping_json) - end - - def update_mapping(mapping_json) - header = { 'Content-type': 'application/json' } - response = RestClient::Request.execute( - method: :put, - url: "#{server_uri}/#{index}/_mapping/#{doctype}", - payload: mapping_json, - headers: header) - end - - def json_to_es(paste_json, method=nil) - header = { 'Content-type': 'application/json' } - response = RestClient::Request.execute( - method: method ||= DEFAULT_METHOD, - url: "#{server_uri}/#{index}/#{doctype}", - headers: header, - payload: paste_json) - end - - def json_to_es_bulk(array_of_paste_json) - array_of_paste_json.each do |paste_json| - self.json_to_es(paste_json) - end - end - end -end diff --git a/lib/pastebinner/option_parser.rb b/lib/pastebinner/option_parser.rb index ab2b120..951e5f3 100644 --- a/lib/pastebinner/option_parser.rb +++ b/lib/pastebinner/option_parser.rb @@ -40,9 +40,6 @@ module Pastebinner options[:j] = true end - opts.on('-t', '--trending', 'Trending pastes') do |_t| - options[:t] = true - end opts.parse! end options diff --git a/lib/pastebinner/workers/paste_to_es.rb b/lib/pastebinner/workers/paste_to_es.rb deleted file mode 100644 index 758ad72..0000000 --- a/lib/pastebinner/workers/paste_to_es.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Pastebinner - class PasteToEs - include Sidekiq::Worker - - sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures - def perform(es_object, pb_object, paste_max) - Logger.new(STDOUT).info("PasteToEs started") - # get public pastes and their keys - pastes = pb_object.scrape_public_pastes(paste_max) - keys = pb_object.get_unique_paste_keys(pastes) - - # build it into json and send it to elasticsearch - json_data = pb_object.json_paste(keys) - es_object.json_to_es_bulk(json_data) - end - end -end diff --git a/mappings/create_mappings.json b/mappings/create_mappings.json deleted file mode 100644 index 06b0cdf..0000000 --- a/mappings/create_mappings.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "settings": { - "number_of_shards": 1 - }, - "mappings": { - "paste": { - "properties": { - "paste_metadata": { - "properties": { - "date": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "expire": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "full_url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "hits": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "scrape_url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "size": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "syntax": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "title": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "user": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "paste_text": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - } diff --git a/mappings/mappings.json b/mappings/mappings.json deleted file mode 100644 index 0f1a304..0000000 --- a/mappings/mappings.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "pastes": { - "mappings": { - "paste": { - "properties": { - "paste_metadata": { - "properties": { - "date": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "expire": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "full_url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "hits": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "scrape_url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "size": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "syntax": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "title": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "user": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "paste_text": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - } -} diff --git a/pastebinner.gemspec b/pastebinner.gemspec index 56c8c76..810a864 100644 --- a/pastebinner.gemspec +++ b/pastebinner.gemspec @@ -33,10 +33,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'bundler', '~> 2.0' spec.add_development_dependency 'rake', '~> 10.0' spec.add_development_dependency 'rspec', '~> 3.0' - spec.add_runtime_dependency 'json', '~> 2.0' + spec.add_runtime_dependency 'rails','~> 6.0' spec.add_runtime_dependency 'pry', '~> 0.11' spec.add_runtime_dependency 'rest-client', '~> 2.0' - spec.add_runtime_dependency 'sidekiq', '~> 5.2.5' - spec.add_runtime_dependency 'redis', '~> 4.1.0' - spec.add_runtime_dependency 'sinatra', '~> 2.0.5' end