bpmcdevitt/pastebinner
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

converting to standalone gem for basic interaction with pastebin api and cmdline client

Browse source
This commit is contained in:
Brendan McDevitt 2019-10-29 21:01:03 -05:00
parent 71a490b7fd
commit 30e014a2f5
13 changed files with 3 additions and 460 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bin/console
View file

@ -1,12 +1,9 @@
#!/usr/bin/env ruby
require 'sidekiq'
require 'sidekiq/api'
require 'bundler/setup'
require 'pastebinner'
require 'pry'
# setup restclient log to the console
# make it into a method to toggle on or off
def rest_client_toggle_log(default=false)
@ -19,14 +16,9 @@ Pastebinner.configure do |config|
config.api_key = ENV['pastebin_api_key']
config.api_username = ENV['pastebin_username']
config.api_password = ENV['pastebin_password']
config.elasticsearch_url = ENV['elasticsearch_url']
config.redis_url = ENV['redis_url']
end
pb = Pastebinner::ApiClient.new(Pastebinner.configuration.api_key, Pastebinner.configuration.api_username, Pastebinner.configuration.api_password)
es = Pastebinner::ElasticSearchHelper.new(Pastebinner.configuration.elasticsearch_url, 'pastes')
# sidekiq worker to send pastes to es
paste_to_es_job = Pastebinner::PasteToEs.new
# you should have a pb object to work with now, have fun in pry!
binding.pry

2
bin/pastebinner
View file

@ -1,6 +1,4 @@
#!/usr/bin/env ruby
require 'sidekiq'
require 'sidekiq/api'
require 'bundler/setup'
require 'pastebinner'

9
config/initializers/sidekiq.rb
View file

@ -1,9 +0,0 @@
# server config
Sidekiq.configure_server do |config|
config.redis = { url: ENV["REDIS_SERVER_URL"] }
end
# client config
Sidekiq.configure_client do |config|
config.redis = { url: ENV["REDIS_SERVER_URL"] }
end

52
config/sidekiq.service
View file

@ -1,52 +0,0 @@
#
# systemd unit file for CentOS 7, Ubuntu 15.04
#
# Customize this file based on your bundler location, app directory, etc.
# Put this in /usr/lib/systemd/system (CentOS) or /lib/systemd/system (Ubuntu).
# Run:
# - systemctl enable sidekiq
# - systemctl {start,stop,restart} sidekiq
#
# This file corresponds to a single Sidekiq process. Add multiple copies
# to run multiple processes (sidekiq-1, sidekiq-2, etc).
#
# See Inspeqtor's Systemd wiki page for more detail about Systemd:
# https://github.com/mperham/inspeqtor/wiki/Systemd
#
[Unit]
Description=sidekiq
# start us only once the network and logging subsystems are available,
# consider adding redis-server.service if Redis is local and systemd-managed.
After=syslog.target network.target
# See these pages for lots of options:
# http://0pointer.de/public/systemd-man/systemd.service.html
# http://0pointer.de/public/systemd-man/systemd.exec.html
[Service]
Type=simple
WorkingDirectory=/opt/myapp/current
# If you use rbenv:
# ExecStart=/bin/bash -lc '/home/deploy/.rbenv/shims/bundle exec sidekiq -e production'
# If you use the system's ruby:
ExecStart=/usr/local/bin/bundle exec sidekiq -e production
User=deploy
Group=deploy
UMask=0002
# Greatly reduce Ruby memory fragmentation and heap usage
# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
Environment=MALLOC_ARENA_MAX=2
# if we crash, restart
RestartSec=1
Restart=on-failure
# output goes to /var/log/syslog
StandardOutput=syslog
StandardError=syslog
# This will default to "bundler" if we don't specify it
SyslogIdentifier=sidekiq
[Install]
WantedBy=multi-user.target

61
examples/sinkiq.rb
View file

@ -1,61 +0,0 @@
# Make sure you have Sinatra installed, then start sidekiq with
# ./bin/sidekiq -r ./examples/sinkiq.rb
# Simply run Sinatra with
# ruby examples/sinkiq.rb
# and then browse to http://localhost:4567
#
require 'sinatra'
require 'sidekiq'
require 'redis'
require 'sidekiq/api'
$redis = Redis.new
class SinatraWorker
include Sidekiq::Worker
def perform(msg="lulz you forgot a msg!")
$redis.lpush("sinkiq-example-messages", msg)
end
end
get '/' do
stats = Sidekiq::Stats.new
@failed = stats.failed
@processed = stats.processed
@messages = $redis.lrange('sinkiq-example-messages', 0, -1)
erb :index
end
post '/msg' do
SinatraWorker.perform_async params[:msg]
redirect to('/')
end
__END__
@@ layout
<html>
<head>
<title>Sinatra + Sidekiq</title>
<body>
<%= yield %>
</body>
</html>
@@ index
<h1>Sinatra + Sidekiq Example</h1>
<h2>Failed: <%= @failed %></h2>
<h2>Processed: <%= @processed %></h2>
<form method="post" action="/msg">
<input type="text" name="msg">
<input type="submit" value="Add Message">
</form>
<a href="/">Refresh page</a>
<h3>Messages</h3>
<% @messages.each do |msg| %>
<p><%= msg %></p>
<% end %>

3
lib/pastebinner.rb
View file

@ -1,6 +1,5 @@
# change this to a config module soon:
require '../config/configuration'
require '../config/initializers/sidekiq'
require 'rest-client'
require 'json'
require 'yaml'
@ -8,8 +7,6 @@ require 'optparse'
require 'pastebinner/version'
require 'pastebinner/option_parser'
require 'pastebinner/api_client'
require 'pastebinner/helpers/elastic_search_helper'
require 'pastebinner/workers/paste_to_es'
module Pastebinner

14
lib/pastebinner/api_client.rb
View file

@ -61,12 +61,6 @@ module Pastebinner
execute_query(:api_post, params)
end
def list_trending_pastes
params = { 'api_dev_key': api_dev_key,
'api_option': 'trends' }
execute_query(:api_post, params)
end
def list_raw_user_paste(api_paste_key)
params = { 'api_dev_key': api_dev_key,
'api_user_key': api_user_key,
@ -111,13 +105,6 @@ module Pastebinner
pp.map { |p| p['key'] }
end
# scraped keys difference returned
# https://stackoverflow.com/questions/8639857/rails-3-how-to-get-the-difference-between-two-arrays
# note ruby 2.6 has a method for this - https://github.com/ruby/ruby/blob/trunk/array.c#L4450-L4563
def difference(a, b)
a - b | b - a
end
def raw_paste_data(unique_paste_key)
response = RestClient::Request.execute(
method: :get,
@ -171,7 +158,6 @@ module Pastebinner
self.hash_paste(raw_paste_json).to_json
end
# keep this method private so we are not letting anyone run any method in our program
private
# this will be the main way to execute any of these methods. this has the exception handling taken care of.

58
lib/pastebinner/helpers/elastic_search_helper.rb
View file

@ -1,58 +0,0 @@
module Pastebinner
class ElasticSearchHelper
attr_accessor :server_uri, :index, :api_client, :doctype
DEFAULT_METHOD = :post
def initialize(server_uri, index, doctype='paste')
@server_uri = server_uri
@index = index
@doctype = doctype
@api_client = Pastebinner::ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
end
def delete_index
response = RestClient::Request.execute(
method: :delete,
url: "#{server_uri}/#{index}")
end
def get_mappings
response = RestClient::Request.execute(
method: :get,
url: "#{server_uri}/#{index}/_mappings?pretty")
end
def create_mapping(mapping_json)
header = { 'Content-type': 'application/json' }
response = RestClient::Request.execute(
method: :put,
url: "#{server_uri}/#{index}",
headers: header,
payload: mapping_json)
end
def update_mapping(mapping_json)
header = { 'Content-type': 'application/json' }
response = RestClient::Request.execute(
method: :put,
url: "#{server_uri}/#{index}/_mapping/#{doctype}",
payload: mapping_json,
headers: header)
end
def json_to_es(paste_json, method=nil)
header = { 'Content-type': 'application/json' }
response = RestClient::Request.execute(
method: method ||= DEFAULT_METHOD,
url: "#{server_uri}/#{index}/#{doctype}",
headers: header,
payload: paste_json)
end
def json_to_es_bulk(array_of_paste_json)
array_of_paste_json.each do |paste_json|
self.json_to_es(paste_json)
end
end
end
end

3
lib/pastebinner/option_parser.rb
View file

@ -40,9 +40,6 @@ module Pastebinner
options[:j] = true
end
opts.on('-t', '--trending', 'Trending pastes') do |_t|
options[:t] = true
end
opts.parse!
end
options

17
lib/pastebinner/workers/paste_to_es.rb
View file

@ -1,17 +0,0 @@
module Pastebinner
class PasteToEs
include Sidekiq::Worker
sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures
def perform(es_object, pb_object, paste_max)
Logger.new(STDOUT).info("PasteToEs started")
# get public pastes and their keys
pastes = pb_object.scrape_public_pastes(paste_max)
keys = pb_object.get_unique_paste_keys(pastes)
# build it into json and send it to elasticsearch
json_data = pb_object.json_paste(keys)
es_object.json_to_es_bulk(json_data)
end
end
end

114
mappings/create_mappings.json
View file

@ -1,114 +0,0 @@
{
"settings": {
"number_of_shards": 1
},
"mappings": {
"paste": {
"properties": {
"paste_metadata": {
"properties": {
"date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"expire": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"full_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"key": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"scrape_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"syntax": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"paste_text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}

113
mappings/mappings.json
View file

@ -1,113 +0,0 @@
{
"pastes": {
"mappings": {
"paste": {
"properties": {
"paste_metadata": {
"properties": {
"date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"expire": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"full_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"key": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"scrape_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"syntax": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"paste_text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}

5
pastebinner.gemspec
View file

@ -33,10 +33,7 @@ Gem::Specification.new do |spec|
spec.add_development_dependency 'bundler', '~> 2.0'
spec.add_development_dependency 'rake', '~> 10.0'
spec.add_development_dependency 'rspec', '~> 3.0'
spec.add_runtime_dependency 'json', '~> 2.0'
spec.add_runtime_dependency 'rails','~> 6.0'
spec.add_runtime_dependency 'pry', '~> 0.11'
spec.add_runtime_dependency 'rest-client', '~> 2.0'
spec.add_runtime_dependency 'sidekiq', '~> 5.2.5'
spec.add_runtime_dependency 'redis', '~> 4.1.0'
spec.add_runtime_dependency 'sinatra', '~> 2.0.5'
end