converting to standalone gem for basic interaction with pastebin api and cmdline client
This commit is contained in:
parent
71a490b7fd
commit
30e014a2f5
13 changed files with 3 additions and 460 deletions
12
bin/console
12
bin/console
|
@ -1,12 +1,9 @@
|
||||||
#!/usr/bin/env ruby
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
require 'sidekiq'
|
|
||||||
require 'sidekiq/api'
|
|
||||||
|
|
||||||
require 'bundler/setup'
|
require 'bundler/setup'
|
||||||
require 'pastebinner'
|
require 'pastebinner'
|
||||||
|
|
||||||
require 'pry'
|
require 'pry'
|
||||||
|
|
||||||
# setup restclient log to the console
|
# setup restclient log to the console
|
||||||
# make it into a method to toggle on or off
|
# make it into a method to toggle on or off
|
||||||
def rest_client_toggle_log(default=false)
|
def rest_client_toggle_log(default=false)
|
||||||
|
@ -19,14 +16,9 @@ Pastebinner.configure do |config|
|
||||||
config.api_key = ENV['pastebin_api_key']
|
config.api_key = ENV['pastebin_api_key']
|
||||||
config.api_username = ENV['pastebin_username']
|
config.api_username = ENV['pastebin_username']
|
||||||
config.api_password = ENV['pastebin_password']
|
config.api_password = ENV['pastebin_password']
|
||||||
config.elasticsearch_url = ENV['elasticsearch_url']
|
|
||||||
config.redis_url = ENV['redis_url']
|
|
||||||
end
|
end
|
||||||
|
|
||||||
pb = Pastebinner::ApiClient.new(Pastebinner.configuration.api_key, Pastebinner.configuration.api_username, Pastebinner.configuration.api_password)
|
pb = Pastebinner::ApiClient.new(Pastebinner.configuration.api_key, Pastebinner.configuration.api_username, Pastebinner.configuration.api_password)
|
||||||
es = Pastebinner::ElasticSearchHelper.new(Pastebinner.configuration.elasticsearch_url, 'pastes')
|
|
||||||
|
|
||||||
# sidekiq worker to send pastes to es
|
|
||||||
paste_to_es_job = Pastebinner::PasteToEs.new
|
|
||||||
|
|
||||||
|
# you should have a pb object to work with now, have fun in pry!
|
||||||
binding.pry
|
binding.pry
|
||||||
|
|
|
@ -1,6 +1,4 @@
|
||||||
#!/usr/bin/env ruby
|
#!/usr/bin/env ruby
|
||||||
require 'sidekiq'
|
|
||||||
require 'sidekiq/api'
|
|
||||||
require 'bundler/setup'
|
require 'bundler/setup'
|
||||||
require 'pastebinner'
|
require 'pastebinner'
|
||||||
|
|
||||||
|
|
|
@ -1,9 +0,0 @@
|
||||||
# server config
|
|
||||||
Sidekiq.configure_server do |config|
|
|
||||||
config.redis = { url: ENV["REDIS_SERVER_URL"] }
|
|
||||||
end
|
|
||||||
|
|
||||||
# client config
|
|
||||||
Sidekiq.configure_client do |config|
|
|
||||||
config.redis = { url: ENV["REDIS_SERVER_URL"] }
|
|
||||||
end
|
|
|
@ -1,52 +0,0 @@
|
||||||
#
|
|
||||||
# systemd unit file for CentOS 7, Ubuntu 15.04
|
|
||||||
#
|
|
||||||
# Customize this file based on your bundler location, app directory, etc.
|
|
||||||
# Put this in /usr/lib/systemd/system (CentOS) or /lib/systemd/system (Ubuntu).
|
|
||||||
# Run:
|
|
||||||
# - systemctl enable sidekiq
|
|
||||||
# - systemctl {start,stop,restart} sidekiq
|
|
||||||
#
|
|
||||||
# This file corresponds to a single Sidekiq process. Add multiple copies
|
|
||||||
# to run multiple processes (sidekiq-1, sidekiq-2, etc).
|
|
||||||
#
|
|
||||||
# See Inspeqtor's Systemd wiki page for more detail about Systemd:
|
|
||||||
# https://github.com/mperham/inspeqtor/wiki/Systemd
|
|
||||||
#
|
|
||||||
[Unit]
|
|
||||||
Description=sidekiq
|
|
||||||
# start us only once the network and logging subsystems are available,
|
|
||||||
# consider adding redis-server.service if Redis is local and systemd-managed.
|
|
||||||
After=syslog.target network.target
|
|
||||||
|
|
||||||
# See these pages for lots of options:
|
|
||||||
# http://0pointer.de/public/systemd-man/systemd.service.html
|
|
||||||
# http://0pointer.de/public/systemd-man/systemd.exec.html
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
WorkingDirectory=/opt/myapp/current
|
|
||||||
# If you use rbenv:
|
|
||||||
# ExecStart=/bin/bash -lc '/home/deploy/.rbenv/shims/bundle exec sidekiq -e production'
|
|
||||||
# If you use the system's ruby:
|
|
||||||
ExecStart=/usr/local/bin/bundle exec sidekiq -e production
|
|
||||||
User=deploy
|
|
||||||
Group=deploy
|
|
||||||
UMask=0002
|
|
||||||
|
|
||||||
# Greatly reduce Ruby memory fragmentation and heap usage
|
|
||||||
# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
|
|
||||||
Environment=MALLOC_ARENA_MAX=2
|
|
||||||
|
|
||||||
# if we crash, restart
|
|
||||||
RestartSec=1
|
|
||||||
Restart=on-failure
|
|
||||||
|
|
||||||
# output goes to /var/log/syslog
|
|
||||||
StandardOutput=syslog
|
|
||||||
StandardError=syslog
|
|
||||||
|
|
||||||
# This will default to "bundler" if we don't specify it
|
|
||||||
SyslogIdentifier=sidekiq
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
|
@ -1,61 +0,0 @@
|
||||||
# Make sure you have Sinatra installed, then start sidekiq with
|
|
||||||
# ./bin/sidekiq -r ./examples/sinkiq.rb
|
|
||||||
# Simply run Sinatra with
|
|
||||||
# ruby examples/sinkiq.rb
|
|
||||||
# and then browse to http://localhost:4567
|
|
||||||
#
|
|
||||||
require 'sinatra'
|
|
||||||
require 'sidekiq'
|
|
||||||
require 'redis'
|
|
||||||
require 'sidekiq/api'
|
|
||||||
|
|
||||||
$redis = Redis.new
|
|
||||||
|
|
||||||
class SinatraWorker
|
|
||||||
include Sidekiq::Worker
|
|
||||||
|
|
||||||
def perform(msg="lulz you forgot a msg!")
|
|
||||||
$redis.lpush("sinkiq-example-messages", msg)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
get '/' do
|
|
||||||
stats = Sidekiq::Stats.new
|
|
||||||
@failed = stats.failed
|
|
||||||
@processed = stats.processed
|
|
||||||
@messages = $redis.lrange('sinkiq-example-messages', 0, -1)
|
|
||||||
erb :index
|
|
||||||
end
|
|
||||||
|
|
||||||
post '/msg' do
|
|
||||||
SinatraWorker.perform_async params[:msg]
|
|
||||||
redirect to('/')
|
|
||||||
end
|
|
||||||
|
|
||||||
__END__
|
|
||||||
|
|
||||||
@@ layout
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Sinatra + Sidekiq</title>
|
|
||||||
<body>
|
|
||||||
<%= yield %>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
||||||
@@ index
|
|
||||||
<h1>Sinatra + Sidekiq Example</h1>
|
|
||||||
<h2>Failed: <%= @failed %></h2>
|
|
||||||
<h2>Processed: <%= @processed %></h2>
|
|
||||||
|
|
||||||
<form method="post" action="/msg">
|
|
||||||
<input type="text" name="msg">
|
|
||||||
<input type="submit" value="Add Message">
|
|
||||||
</form>
|
|
||||||
|
|
||||||
<a href="/">Refresh page</a>
|
|
||||||
|
|
||||||
<h3>Messages</h3>
|
|
||||||
<% @messages.each do |msg| %>
|
|
||||||
<p><%= msg %></p>
|
|
||||||
<% end %>
|
|
|
@ -1,6 +1,5 @@
|
||||||
# change this to a config module soon:
|
# change this to a config module soon:
|
||||||
require '../config/configuration'
|
require '../config/configuration'
|
||||||
require '../config/initializers/sidekiq'
|
|
||||||
require 'rest-client'
|
require 'rest-client'
|
||||||
require 'json'
|
require 'json'
|
||||||
require 'yaml'
|
require 'yaml'
|
||||||
|
@ -8,8 +7,6 @@ require 'optparse'
|
||||||
require 'pastebinner/version'
|
require 'pastebinner/version'
|
||||||
require 'pastebinner/option_parser'
|
require 'pastebinner/option_parser'
|
||||||
require 'pastebinner/api_client'
|
require 'pastebinner/api_client'
|
||||||
require 'pastebinner/helpers/elastic_search_helper'
|
|
||||||
require 'pastebinner/workers/paste_to_es'
|
|
||||||
|
|
||||||
module Pastebinner
|
module Pastebinner
|
||||||
|
|
||||||
|
|
|
@ -61,12 +61,6 @@ module Pastebinner
|
||||||
execute_query(:api_post, params)
|
execute_query(:api_post, params)
|
||||||
end
|
end
|
||||||
|
|
||||||
def list_trending_pastes
|
|
||||||
params = { 'api_dev_key': api_dev_key,
|
|
||||||
'api_option': 'trends' }
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
|
||||||
|
|
||||||
def list_raw_user_paste(api_paste_key)
|
def list_raw_user_paste(api_paste_key)
|
||||||
params = { 'api_dev_key': api_dev_key,
|
params = { 'api_dev_key': api_dev_key,
|
||||||
'api_user_key': api_user_key,
|
'api_user_key': api_user_key,
|
||||||
|
@ -111,13 +105,6 @@ module Pastebinner
|
||||||
pp.map { |p| p['key'] }
|
pp.map { |p| p['key'] }
|
||||||
end
|
end
|
||||||
|
|
||||||
# scraped keys difference returned
|
|
||||||
# https://stackoverflow.com/questions/8639857/rails-3-how-to-get-the-difference-between-two-arrays
|
|
||||||
# note ruby 2.6 has a method for this - https://github.com/ruby/ruby/blob/trunk/array.c#L4450-L4563
|
|
||||||
def difference(a, b)
|
|
||||||
a - b | b - a
|
|
||||||
end
|
|
||||||
|
|
||||||
def raw_paste_data(unique_paste_key)
|
def raw_paste_data(unique_paste_key)
|
||||||
response = RestClient::Request.execute(
|
response = RestClient::Request.execute(
|
||||||
method: :get,
|
method: :get,
|
||||||
|
@ -171,7 +158,6 @@ module Pastebinner
|
||||||
self.hash_paste(raw_paste_json).to_json
|
self.hash_paste(raw_paste_json).to_json
|
||||||
end
|
end
|
||||||
|
|
||||||
# keep this method private so we are not letting anyone run any method in our program
|
|
||||||
private
|
private
|
||||||
|
|
||||||
# this will be the main way to execute any of these methods. this has the exception handling taken care of.
|
# this will be the main way to execute any of these methods. this has the exception handling taken care of.
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
module Pastebinner
|
|
||||||
class ElasticSearchHelper
|
|
||||||
attr_accessor :server_uri, :index, :api_client, :doctype
|
|
||||||
DEFAULT_METHOD = :post
|
|
||||||
|
|
||||||
def initialize(server_uri, index, doctype='paste')
|
|
||||||
@server_uri = server_uri
|
|
||||||
@index = index
|
|
||||||
@doctype = doctype
|
|
||||||
@api_client = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
|
||||||
end
|
|
||||||
|
|
||||||
def delete_index
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :delete,
|
|
||||||
url: "#{server_uri}/#{index}")
|
|
||||||
end
|
|
||||||
|
|
||||||
def get_mappings
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :get,
|
|
||||||
url: "#{server_uri}/#{index}/_mappings?pretty")
|
|
||||||
end
|
|
||||||
|
|
||||||
def create_mapping(mapping_json)
|
|
||||||
header = { 'Content-type': 'application/json' }
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :put,
|
|
||||||
url: "#{server_uri}/#{index}",
|
|
||||||
headers: header,
|
|
||||||
payload: mapping_json)
|
|
||||||
end
|
|
||||||
|
|
||||||
def update_mapping(mapping_json)
|
|
||||||
header = { 'Content-type': 'application/json' }
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :put,
|
|
||||||
url: "#{server_uri}/#{index}/_mapping/#{doctype}",
|
|
||||||
payload: mapping_json,
|
|
||||||
headers: header)
|
|
||||||
end
|
|
||||||
|
|
||||||
def json_to_es(paste_json, method=nil)
|
|
||||||
header = { 'Content-type': 'application/json' }
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: method ||= DEFAULT_METHOD,
|
|
||||||
url: "#{server_uri}/#{index}/#{doctype}",
|
|
||||||
headers: header,
|
|
||||||
payload: paste_json)
|
|
||||||
end
|
|
||||||
|
|
||||||
def json_to_es_bulk(array_of_paste_json)
|
|
||||||
array_of_paste_json.each do |paste_json|
|
|
||||||
self.json_to_es(paste_json)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -40,9 +40,6 @@ module Pastebinner
|
||||||
options[:j] = true
|
options[:j] = true
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-t', '--trending', 'Trending pastes') do |_t|
|
|
||||||
options[:t] = true
|
|
||||||
end
|
|
||||||
opts.parse!
|
opts.parse!
|
||||||
end
|
end
|
||||||
options
|
options
|
||||||
|
|
|
@ -1,17 +0,0 @@
|
||||||
module Pastebinner
|
|
||||||
class PasteToEs
|
|
||||||
include Sidekiq::Worker
|
|
||||||
|
|
||||||
sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures
|
|
||||||
def perform(es_object, pb_object, paste_max)
|
|
||||||
Logger.new(STDOUT).info("PasteToEs started")
|
|
||||||
# get public pastes and their keys
|
|
||||||
pastes = pb_object.scrape_public_pastes(paste_max)
|
|
||||||
keys = pb_object.get_unique_paste_keys(pastes)
|
|
||||||
|
|
||||||
# build it into json and send it to elasticsearch
|
|
||||||
json_data = pb_object.json_paste(keys)
|
|
||||||
es_object.json_to_es_bulk(json_data)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -1,114 +0,0 @@
|
||||||
{
|
|
||||||
"settings": {
|
|
||||||
"number_of_shards": 1
|
|
||||||
},
|
|
||||||
"mappings": {
|
|
||||||
"paste": {
|
|
||||||
"properties": {
|
|
||||||
"paste_metadata": {
|
|
||||||
"properties": {
|
|
||||||
"date": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"expire": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"full_url": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"hits": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"key": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"scrape_url": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"size": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"syntax": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"title": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"user": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"paste_text": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,113 +0,0 @@
|
||||||
{
|
|
||||||
"pastes": {
|
|
||||||
"mappings": {
|
|
||||||
"paste": {
|
|
||||||
"properties": {
|
|
||||||
"paste_metadata": {
|
|
||||||
"properties": {
|
|
||||||
"date": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"expire": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"full_url": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"hits": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"key": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"scrape_url": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"size": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"syntax": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"title": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"user": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"paste_text": {
|
|
||||||
"type": "text",
|
|
||||||
"fields": {
|
|
||||||
"keyword": {
|
|
||||||
"type": "keyword",
|
|
||||||
"ignore_above": 256
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -33,10 +33,7 @@ Gem::Specification.new do |spec|
|
||||||
spec.add_development_dependency 'bundler', '~> 2.0'
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
||||||
spec.add_development_dependency 'rake', '~> 10.0'
|
spec.add_development_dependency 'rake', '~> 10.0'
|
||||||
spec.add_development_dependency 'rspec', '~> 3.0'
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
||||||
spec.add_runtime_dependency 'json', '~> 2.0'
|
spec.add_runtime_dependency 'rails','~> 6.0'
|
||||||
spec.add_runtime_dependency 'pry', '~> 0.11'
|
spec.add_runtime_dependency 'pry', '~> 0.11'
|
||||||
spec.add_runtime_dependency 'rest-client', '~> 2.0'
|
spec.add_runtime_dependency 'rest-client', '~> 2.0'
|
||||||
spec.add_runtime_dependency 'sidekiq', '~> 5.2.5'
|
|
||||||
spec.add_runtime_dependency 'redis', '~> 4.1.0'
|
|
||||||
spec.add_runtime_dependency 'sinatra', '~> 2.0.5'
|
|
||||||
end
|
end
|
||||||
|
|
Loading…
Add table
Reference in a new issue