made elastic search helper better and made pastebinner pack json
This commit is contained in:
parent
ba20e8dc41
commit
50b052a9bd
14 changed files with 256 additions and 296 deletions
4
Gemfile
4
Gemfile
|
@ -1,6 +1,6 @@
|
||||||
source "https://rubygems.org"
|
source 'https://rubygems.org'
|
||||||
|
|
||||||
git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
|
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
||||||
|
|
||||||
# Specify your gem's dependencies in pastebinner.gemspec
|
# Specify your gem's dependencies in pastebinner.gemspec
|
||||||
gemspec
|
gemspec
|
||||||
|
|
6
Rakefile
6
Rakefile
|
@ -1,6 +1,6 @@
|
||||||
require "bundler/gem_tasks"
|
require 'bundler/gem_tasks'
|
||||||
require "rspec/core/rake_task"
|
require 'rspec/core/rake_task'
|
||||||
|
|
||||||
RSpec::Core::RakeTask.new(:spec)
|
RSpec::Core::RakeTask.new(:spec)
|
||||||
|
|
||||||
task :default => :spec
|
task default: :spec
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
#!/usr/bin/env ruby
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
require "../lib/pastebinner"
|
require '../lib/pastebinner'
|
||||||
|
require '../lib/elastic_search_helper'
|
||||||
|
|
||||||
# You can add fixtures and/or initialization code here to make experimenting
|
# You can add fixtures and/or initialization code here to make experimenting
|
||||||
# with your gem easier. You can also use a different console, if you like.
|
# with your gem easier. You can also use a different console, if you like.
|
||||||
|
|
||||||
# (If you use this, don't forget to add pry to your Gemfile!)
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
||||||
require "pry"
|
require 'pry'
|
||||||
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||||||
binding.pry
|
binding.pry
|
||||||
|
|
|
@ -4,15 +4,15 @@ require '../lib/pastebinner/option_parser'
|
||||||
require 'pry'
|
require 'pry'
|
||||||
|
|
||||||
# setup our object and grab a session key
|
# setup our object and grab a session key
|
||||||
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||||||
|
|
||||||
# pass in the Pastebinner.new client.
|
# pass in the Pastebinner.new client.
|
||||||
# will download all of the raw pastes from the public scrape results into each own file in data dir.
|
# will download all of the raw pastes from the public scrape results into each own file in data dir.
|
||||||
def download_pastes(pb)
|
def download_pastes(pb)
|
||||||
pub_pastes = pb.scrape_public_pastes
|
pub_pastes = pb.scrape_public_pastes
|
||||||
keys = pb.get_unique_paste_keys(pub_pastes)
|
keys = pb.get_unique_paste_keys(pub_pastes)
|
||||||
data_dir = "../data/"
|
data_dir = '../data/'
|
||||||
filename = "pastebin_paste_key"
|
filename = 'pastebin_paste_key'
|
||||||
binding.pry
|
binding.pry
|
||||||
keys.map do |id|
|
keys.map do |id|
|
||||||
if File.exist?(data_dir + filename + "_#{id}")
|
if File.exist?(data_dir + filename + "_#{id}")
|
||||||
|
@ -27,7 +27,7 @@ options = OptionParser.parse!
|
||||||
|
|
||||||
if options[:s]
|
if options[:s]
|
||||||
puts pb.scrape_public_pastes
|
puts pb.scrape_public_pastes
|
||||||
elsif options[:r] and options[:k]
|
elsif options[:r] && options[:k]
|
||||||
key = options[:k]
|
key = options[:k]
|
||||||
puts pb.raw_paste_data(key)
|
puts pb.raw_paste_data(key)
|
||||||
elsif options[:t]
|
elsif options[:t]
|
||||||
|
@ -36,13 +36,13 @@ elsif options[:g]
|
||||||
r = pb.scrape_public_pastes
|
r = pb.scrape_public_pastes
|
||||||
puts pb.get_unique_paste_keys(r)
|
puts pb.get_unique_paste_keys(r)
|
||||||
elsif options[:d]
|
elsif options[:d]
|
||||||
puts "Downloading paste data into the data directory..."
|
puts 'Downloading paste data into the data directory...'
|
||||||
download_pastes(pb)
|
download_pastes(pb)
|
||||||
puts "Complete."
|
puts 'Complete.'
|
||||||
elsif options[:k]
|
elsif options[:k]
|
||||||
puts '-k or --key= requires -r,--raw'
|
puts '-k or --key= requires -r,--raw'
|
||||||
exit
|
exit
|
||||||
else options = false
|
else options = false
|
||||||
puts 'please provide arguments'
|
puts 'please provide arguments'
|
||||||
exit
|
exit
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,53 +1,33 @@
|
||||||
require 'elasticsearch'
|
|
||||||
|
|
||||||
class ElasticSearchHelper
|
class ElasticSearchHelper
|
||||||
attr_accessor :server_uri, :index
|
attr_accessor :server_uri, :index, :pastebinner, :doctype
|
||||||
|
DEFAULT_METHOD = :post
|
||||||
|
|
||||||
def initialize(server_uri, index)
|
def initialize(server_uri, index, doctype='_doc')
|
||||||
@server_uri = server_uri
|
@server_uri = server_uri
|
||||||
@index = index
|
@index = index
|
||||||
|
@doctype = doctype
|
||||||
|
@pastebinner = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||||||
end
|
end
|
||||||
|
|
||||||
# will build an array of 50 pastes to ship to es
|
def create_index
|
||||||
def build_json_array(pb, keys)
|
response = RestClient::Request.execute(
|
||||||
json_for_es = keys.map do |k|
|
method: :put,
|
||||||
pb.encode_json(pb.raw_paste_data(k), pb.raw_paste_metadata(k))
|
url: "#{server_uri}/#{index}")
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_to_es(paste_json, method=nil)
|
||||||
|
header = { 'Content-type': 'application/json' }
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: method ||= DEFAULT_METHOD,
|
||||||
|
url: "#{server_uri}/#{index}/#{doctype}",
|
||||||
|
headers: header,
|
||||||
|
payload: paste_json)
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_to_es_bulk(array_of_paste_json)
|
||||||
|
array_of_paste_json.each do |paste_json|
|
||||||
|
self.to_es(paste_json)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def puts_to_es(payload, increment_num)
|
|
||||||
header = { 'Content-type': 'application/json' }
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :put,
|
|
||||||
url: "#{server_uri}/#{index}/#{index}s/#{increment_num}",
|
|
||||||
headers: header,
|
|
||||||
payload: payload)
|
|
||||||
end
|
|
||||||
|
|
||||||
def data_mappings
|
|
||||||
# metadata mappings
|
|
||||||
# send a PUT
|
|
||||||
{
|
|
||||||
"mappings": {
|
|
||||||
"_doc": {
|
|
||||||
"properties": {
|
|
||||||
"type": { "type": "keyword" },
|
|
||||||
"paste_metadata": { "type": "nested" },
|
|
||||||
"properties": {
|
|
||||||
"scrape_url": { "type": "string" },
|
|
||||||
"full_url": { "type": "string" },
|
|
||||||
"date": { "type": "string" },
|
|
||||||
"size": { "type": "string" },
|
|
||||||
"expire": { "type": "string" },
|
|
||||||
"title": { "type": "string" },
|
|
||||||
"syntax": { "type": "string" },
|
|
||||||
"user": { "type": "string" },
|
|
||||||
"hits": { "type": "string" }
|
|
||||||
}
|
|
||||||
"paste_text": { "type": "string" }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
4
lib/examples/examples.rb
Normal file → Executable file
4
lib/examples/examples.rb
Normal file → Executable file
|
@ -8,14 +8,14 @@ require '../pastebinner'
|
||||||
#### INITIAL STEPS
|
#### INITIAL STEPS
|
||||||
|
|
||||||
# setup our object and grab a session key
|
# setup our object and grab a session key
|
||||||
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||||||
api_dev_key = ENV['pastebin_api_key']
|
api_dev_key = ENV['pastebin_api_key']
|
||||||
|
|
||||||
#### CREATE PASTE
|
#### CREATE PASTE
|
||||||
# prepare some sample paste data to send
|
# prepare some sample paste data to send
|
||||||
paste_data = 'this is a test paste two two two.'
|
paste_data = 'this is a test paste two two two.'
|
||||||
# prepare our paste params
|
# prepare our paste params
|
||||||
params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data }
|
params = { "api_dev_key": api_dev_key, "api_option": 'paste', "api_paste_code": paste_data }
|
||||||
puts pb.create_paste(params)
|
puts pb.create_paste(params)
|
||||||
|
|
||||||
#### SCRAPE PUBLIC PASTES
|
#### SCRAPE PUBLIC PASTES
|
||||||
|
|
|
@ -1,16 +1,13 @@
|
||||||
module PastebinnerError
|
module PastebinnerError
|
||||||
|
|
||||||
class ArgumentError < StandardError
|
class ArgumentError < StandardError
|
||||||
def message
|
def message
|
||||||
"Invalid argument"
|
'Invalid argument'
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
class ConfigError < StandardError
|
class ConfigError < StandardError
|
||||||
def message
|
def message
|
||||||
"Invalid configuration"
|
'Invalid configuration'
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -6,202 +6,189 @@ require 'rest-client'
|
||||||
require 'json'
|
require 'json'
|
||||||
|
|
||||||
class Pastebinner
|
class Pastebinner
|
||||||
attr_accessor :api_dev_key, :username, :password
|
attr_accessor :api_dev_key, :username, :password
|
||||||
|
|
||||||
def initialize(api_dev_key, username, password)
|
def initialize(api_dev_key, username, password)
|
||||||
@api_dev_key = api_dev_key
|
@api_dev_key = api_dev_key
|
||||||
@username = username
|
@username = username
|
||||||
@password = password
|
@password = password
|
||||||
@base_api_url = 'https://pastebin.com/api'
|
@base_api_url = 'https://pastebin.com/api'
|
||||||
@scraping_api_url = 'https://scrape.pastebin.com'
|
@scraping_api_url = 'https://scrape.pastebin.com'
|
||||||
|
end
|
||||||
|
|
||||||
|
# this should be a hash of { endpoint_name: '/url_endpoint.php'}
|
||||||
|
ENDPOINTS = { login: '/api_login.php',
|
||||||
|
post: '/api_post.php',
|
||||||
|
raw: '/api_raw.php',
|
||||||
|
scraping: '/api_scraping.php',
|
||||||
|
scrape_item: '/api_scrape_item.php',
|
||||||
|
scrape_item_meta: '/api_scrape_item_meta.php' }.freeze
|
||||||
|
|
||||||
|
# basic example hash for creating a paste:
|
||||||
|
# params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data}
|
||||||
|
|
||||||
|
# required params:
|
||||||
|
# api_dev_key - your unique developer api key
|
||||||
|
# api_option - set as paste, this will indicate you want to create a new paste
|
||||||
|
# api_paste_code - this is the text that will be written inside of your paste
|
||||||
|
|
||||||
|
# optional params:
|
||||||
|
# api_user_key - this parameter is part of the login system, which is explained further down the page
|
||||||
|
# api_paste_name - this will be the name / title of your paste
|
||||||
|
# api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page
|
||||||
|
# api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2
|
||||||
|
# api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page
|
||||||
|
|
||||||
|
# example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data }
|
||||||
|
def create_paste(params)
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def api_user_key
|
||||||
|
# returns a user session key that can be used as the api_user_key param
|
||||||
|
@api_user_key ||= RestClient::Request.execute(
|
||||||
|
method: :post,
|
||||||
|
url: @base_api_url + ENDPOINTS[:login],
|
||||||
|
payload: { 'api_dev_key': @api_dev_key,
|
||||||
|
'api_user_name': @username,
|
||||||
|
'api_user_password': @password }
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def list_user_pastes
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_results_limit': '100',
|
||||||
|
'api_option': 'list' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def list_trending_pastes
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_option': 'trends' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def list_raw_user_paste(api_paste_key)
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_paste_key': api_paste_key,
|
||||||
|
'api_option': 'show_paste' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
# api_paste_key = this is the unique key of the paste data you want to delete.
|
||||||
|
def delete_user_paste(api_paste_key)
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_paste_key': api_paste_key,
|
||||||
|
'api_option': 'delete' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_user_info
|
||||||
|
params = { 'api_dev_key': api_dev_key }
|
||||||
|
end
|
||||||
|
|
||||||
|
def api_post(params)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :post,
|
||||||
|
url: @base_api_url + ENDPOINTS[:post],
|
||||||
|
payload: params
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
# params is optional for now. to query specific language ?lang=ruby as an example
|
||||||
|
def scrape_public_pastes(_params = nil)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scraping]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
# will extract just the keys from recent public pastes
|
||||||
|
def get_unique_paste_keys(public_pastes)
|
||||||
|
pp = JSON.parse(public_pastes)
|
||||||
|
pp.map { |p| p['key'] }
|
||||||
|
end
|
||||||
|
|
||||||
|
def raw_paste_data(unique_paste_key)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def raw_paste_metadata(unique_paste_key)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}"
|
||||||
|
)
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
##### PREPARING THE PASTES FOR SERIALIZATION FOR ES CONFORMING TO PER INDEX SEARCHING
|
||||||
|
##### SEE - https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#_custom_type_field
|
||||||
|
|
||||||
|
def hash_pastes(keys)
|
||||||
|
keys.map do |key|
|
||||||
|
raw_paste = self.raw_paste_data(key).body
|
||||||
|
raw_paste_metadata = self.raw_paste_metadata(key).body
|
||||||
|
hash = self.hash_paste(raw_paste, raw_paste_metadata)
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
# this should be a hash of { endpoint_name: '/url_endpoint.php'}
|
def json_pastes(keys)
|
||||||
ENDPOINTS = { :login => '/api_login.php',
|
self.hash_pastes(keys).map do |paste_hash|
|
||||||
:post => '/api_post.php',
|
paste_hash.to_json
|
||||||
:raw => '/api_raw.php',
|
|
||||||
:scraping => '/api_scraping.php',
|
|
||||||
:scrape_item => '/api_scrape_item.php',
|
|
||||||
:scrape_item_meta => '/api_scrape_item_meta.php' }
|
|
||||||
|
|
||||||
# basic example hash for creating a paste:
|
|
||||||
# params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data}
|
|
||||||
|
|
||||||
# required params:
|
|
||||||
# api_dev_key - your unique developer api key
|
|
||||||
# api_option - set as paste, this will indicate you want to create a new paste
|
|
||||||
# api_paste_code - this is the text that will be written inside of your paste
|
|
||||||
|
|
||||||
# optional params:
|
|
||||||
# api_user_key - this parameter is part of the login system, which is explained further down the page
|
|
||||||
# api_paste_name - this will be the name / title of your paste
|
|
||||||
# api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page
|
|
||||||
# api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2
|
|
||||||
# api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page
|
|
||||||
|
|
||||||
# example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data }
|
|
||||||
def create_paste(params)
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def api_user_key
|
def hash_paste(raw_paste_data, raw_paste_metadata)
|
||||||
# returns a user session key that can be used as the api_user_key param
|
{ "paste_metadata": raw_paste_metadata,
|
||||||
@api_user_key ||= RestClient::Request.execute({
|
"paste_text": raw_paste_data }
|
||||||
method: :post,
|
end
|
||||||
url: @base_api_url + ENDPOINTS[:login],
|
|
||||||
payload: { 'api_dev_key': @api_dev_key,
|
|
||||||
'api_user_name': @username,
|
|
||||||
'api_user_password': @password }})
|
|
||||||
end
|
|
||||||
|
|
||||||
def list_user_pastes
|
def json_paste(raw_paste_data, raw_paste_metadata)
|
||||||
params = { 'api_dev_key': api_dev_key,
|
self.hash_paste(raw_paste_data, raw_paste_metadata).to_json
|
||||||
'api_user_key': api_user_key,
|
end
|
||||||
'api_results_limit': '100',
|
|
||||||
'api_option': 'list'
|
|
||||||
}
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
|
||||||
|
|
||||||
def list_trending_pastes
|
def data_mappings
|
||||||
params = { 'api_dev_key': api_dev_key,
|
|
||||||
'api_option': 'trends'
|
|
||||||
}
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
|
||||||
|
|
||||||
def list_raw_user_paste(api_paste_key)
|
|
||||||
params = { 'api_dev_key': api_dev_key,
|
|
||||||
'api_user_key': api_user_key,
|
|
||||||
'api_paste_key': api_paste_key,
|
|
||||||
'api_option': 'show_paste'
|
|
||||||
}
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
|
||||||
|
|
||||||
# api_paste_key = this is the unique key of the paste data you want to delete.
|
|
||||||
def delete_user_paste(api_paste_key)
|
|
||||||
params = { 'api_dev_key': api_dev_key,
|
|
||||||
'api_user_key': api_user_key,
|
|
||||||
'api_paste_key': api_paste_key,
|
|
||||||
'api_option': 'delete'
|
|
||||||
}
|
|
||||||
execute_query(:api_post, params)
|
|
||||||
end
|
|
||||||
|
|
||||||
def get_user_info
|
|
||||||
params = { 'api_dev_key': api_dev_key,
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
def api_post(params)
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :post,
|
|
||||||
url: @base_api_url + ENDPOINTS[:post],
|
|
||||||
payload: params)
|
|
||||||
end
|
|
||||||
|
|
||||||
# params is optional for now. to query specific language ?lang=ruby as an example
|
|
||||||
def scrape_public_pastes(params = nil)
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :get,
|
|
||||||
url: @scraping_api_url + ENDPOINTS[:scraping])
|
|
||||||
end
|
|
||||||
|
|
||||||
# will extract just the keys from recent public pastes
|
|
||||||
def get_unique_paste_keys(public_pastes)
|
|
||||||
pp = JSON.parse(public_pastes)
|
|
||||||
pp.map {|p| p['key']}
|
|
||||||
end
|
|
||||||
|
|
||||||
def raw_paste_data(unique_paste_key)
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :get,
|
|
||||||
url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}")
|
|
||||||
end
|
|
||||||
|
|
||||||
def raw_paste_metadata(unique_paste_key)
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :get,
|
|
||||||
url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}")
|
|
||||||
response
|
|
||||||
end
|
|
||||||
|
|
||||||
##### PREPARING THE PASTES FOR SERIALIZATION FOR ES CONFORMING TO PER INDEX SEARCHING
|
|
||||||
##### SEE - https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#_custom_type_field
|
|
||||||
|
|
||||||
def http_to_es(es_uri, payload, method)
|
|
||||||
header = { 'Content-type': 'application/json' }
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: method,
|
|
||||||
url: es_uri,
|
|
||||||
headers: header,
|
|
||||||
payload: payload)
|
|
||||||
end
|
|
||||||
|
|
||||||
def create_index_es(es_uri, index)
|
|
||||||
response = RestClient::Request.execute(
|
|
||||||
method: :put,
|
|
||||||
url: es_uri + index)
|
|
||||||
end
|
|
||||||
|
|
||||||
def hash_pastes(keys)
|
|
||||||
keys.map do |key|
|
|
||||||
raw_paste = self.raw_paste_data(key).body
|
|
||||||
raw_paste_metadata = self.raw_paste_metadata(key).body
|
|
||||||
hash = self.hash_paste(raw_paste, raw_paste_metadata)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def hash_paste(raw_paste_data, raw_paste_metadata)
|
|
||||||
{ "paste_metadata": raw_paste_metadata,
|
|
||||||
"paste_text": raw_paste_data }
|
|
||||||
end
|
|
||||||
|
|
||||||
def send_es_bulk(esi_uri, json_data)
|
|
||||||
method = :post
|
|
||||||
json_data.each do |payload|
|
|
||||||
self.http_to_es(esi_uri, payload, method)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def data_mappings
|
|
||||||
# metadata mappings
|
# metadata mappings
|
||||||
# send a PUT
|
# send a PUT
|
||||||
{
|
{
|
||||||
"mappings": {
|
"mappings": {
|
||||||
"_doc": {
|
"_doc": {
|
||||||
"properties": {
|
"properties": {
|
||||||
"type": { "type": "keyword" },
|
"type": { "type": 'keyword' },
|
||||||
"paste_metadata": { "type": "nested" },
|
"paste_metadata": { "type": 'nested' },
|
||||||
"properties": [ {
|
"properties": [{
|
||||||
"scrape_url": { "type": "string" },
|
"scrape_url": { "type": 'string' },
|
||||||
"full_url": { "type": "string" },
|
"full_url": { "type": 'string' },
|
||||||
"date": { "type": "string" },
|
"date": { "type": 'string' },
|
||||||
"size": { "type": "string" },
|
"size": { "type": 'string' },
|
||||||
"expire": { "type": "string" },
|
"expire": { "type": 'string' },
|
||||||
"title": { "type": "string" },
|
"title": { "type": 'string' },
|
||||||
"syntax": { "type": "string" },
|
"syntax": { "type": 'string' },
|
||||||
"user": { "type": "string" },
|
"user": { "type": 'string' },
|
||||||
"hits": { "type": "string" }
|
"hits": { "type": 'string' }
|
||||||
} ],
|
}],
|
||||||
"paste_text": { "type": "string" }
|
"paste_text": { "type": 'string' }
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
end
|
}
|
||||||
|
end
|
||||||
|
|
||||||
# keep this method private so we are not letting anyone run any method in our program
|
# keep this method private so we are not letting anyone run any method in our program
|
||||||
private
|
private
|
||||||
# this will be the main way to execute any of these methods. this has the exception handling taken care of.
|
|
||||||
def execute_query(selector, *args)
|
# this will be the main way to execute any of these methods. this has the exception handling taken care of.
|
||||||
begin
|
def execute_query(selector, *args)
|
||||||
send(selector, *args)
|
send(selector, *args)
|
||||||
rescue RestClient::ExceptionWithResponse => e
|
rescue RestClient::ExceptionWithResponse => e
|
||||||
puts e.message
|
puts e.message
|
||||||
end
|
end
|
||||||
end
|
# make my own exception class
|
||||||
# make my own exception class
|
# inherit ruby standard error class
|
||||||
# inherit ruby standard error class
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,9 +1,5 @@
|
||||||
class PastebinnerError < StandardError
|
class PastebinnerError < StandardError
|
||||||
|
def InvalidArgument; end
|
||||||
|
|
||||||
def InvalidArgument
|
def ConfigError; end
|
||||||
end
|
|
||||||
|
|
||||||
def ConfigError
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -6,22 +6,22 @@ class OptionParser
|
||||||
OptParse.new do |opts|
|
OptParse.new do |opts|
|
||||||
opts.default_argv = argv
|
opts.default_argv = argv
|
||||||
|
|
||||||
opts.banner = "Usage: pastebinner [options]"
|
opts.banner = 'Usage: pastebinner [options]'
|
||||||
|
|
||||||
opts.on('-h', '--help', 'Show this help messae') do ||
|
opts.on('-h', '--help', 'Show this help messae') do
|
||||||
puts opts
|
puts opts
|
||||||
exit
|
exit
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-s', '--scrape_public', 'Scrape public pastes') do |s|
|
opts.on('-s', '--scrape_public', 'Scrape public pastes') do |_s|
|
||||||
options[:s] = true
|
options[:s] = true
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-r', '--raw', 'Raw paste. Requires --key passed with a valid key') do |r|
|
opts.on('-r', '--raw', 'Raw paste. Requires --key passed with a valid key') do |_r|
|
||||||
options[:r] = true
|
options[:r] = true
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-g', '--get_keys', 'Get unique paste keys from public pastes') do |g|
|
opts.on('-g', '--get_keys', 'Get unique paste keys from public pastes') do |_g|
|
||||||
options[:g] = true
|
options[:g] = true
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -29,11 +29,11 @@ class OptionParser
|
||||||
options[:k] = k
|
options[:k] = k
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-d', '--download', 'Download all public pastes to data directory') do |d|
|
opts.on('-d', '--download', 'Download all public pastes to data directory') do |_d|
|
||||||
options[:d] = true
|
options[:d] = true
|
||||||
end
|
end
|
||||||
|
|
||||||
opts.on('-t', '--trending', 'Trending pastes') do |t|
|
opts.on('-t', '--trending', 'Trending pastes') do |_t|
|
||||||
options[:t] = true
|
options[:t] = true
|
||||||
end
|
end
|
||||||
opts.parse!
|
opts.parse!
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
module Pastebinner
|
module Pastebinner
|
||||||
VERSION = "0.1.0"
|
VERSION = '0.1.0'.freeze
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,40 +1,39 @@
|
||||||
lib = File.expand_path("../lib", __FILE__)
|
lib = File.expand_path('lib', __dir__)
|
||||||
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
||||||
require "pastebinner/version"
|
require 'pastebinner/version'
|
||||||
|
|
||||||
Gem::Specification.new do |spec|
|
Gem::Specification.new do |spec|
|
||||||
spec.name = "pastebinner"
|
spec.name = 'pastebinner'
|
||||||
spec.version = Pastebinner::VERSION
|
spec.version = Pastebinner::VERSION
|
||||||
spec.authors = ["Brendan McDevitt"]
|
spec.authors = ['Brendan McDevitt']
|
||||||
spec.email = ["brendan@mcdevitt.tech"]
|
spec.email = ['brendan@mcdevitt.tech']
|
||||||
|
|
||||||
spec.summary = "A ruby client library for interacting with the pastebin API."
|
spec.summary = 'A ruby client library for interacting with the pastebin API.'
|
||||||
spec.homepage = "https://git.mcdevitt.tech/bpmcdevitt/pastebinner"
|
spec.homepage = 'https://git.mcdevitt.tech/bpmcdevitt/pastebinner'
|
||||||
|
|
||||||
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
||||||
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
||||||
if spec.respond_to?(:metadata)
|
if spec.respond_to?(:metadata)
|
||||||
spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
|
||||||
else
|
else
|
||||||
raise "RubyGems 2.0 or newer is required to protect against " \
|
raise 'RubyGems 2.0 or newer is required to protect against ' \
|
||||||
"public gem pushes."
|
'public gem pushes.'
|
||||||
end
|
end
|
||||||
|
|
||||||
# Specify which files should be added to the gem when it is released.
|
# Specify which files should be added to the gem when it is released.
|
||||||
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
||||||
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
||||||
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
||||||
end
|
end
|
||||||
spec.bindir = "bin"
|
spec.bindir = 'bin'
|
||||||
spec.executables = ['pastebinner']
|
spec.executables = ['pastebinner']
|
||||||
|
|
||||||
spec.require_paths = ["lib"]
|
spec.require_paths = ['lib']
|
||||||
|
|
||||||
spec.add_development_dependency "bundler", "~> 2.0"
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
||||||
spec.add_development_dependency "rake", "~> 10.0"
|
spec.add_development_dependency 'rake', '~> 10.0'
|
||||||
spec.add_development_dependency "rspec", "~> 3.0"
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
||||||
spec.add_runtime_dependency "rest-client", "~> 2.0"
|
spec.add_runtime_dependency 'json', '~> 2.0'
|
||||||
spec.add_runtime_dependency "json", "~> 2.0"
|
spec.add_runtime_dependency 'pry', '~> 0.11'
|
||||||
spec.add_runtime_dependency "pry", "~> 0.11"
|
spec.add_runtime_dependency 'rest-client', '~> 2.0'
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
RSpec.describe Pastebinner do
|
RSpec.describe Pastebinner do
|
||||||
it "has a version number" do
|
it 'has a version number' do
|
||||||
expect(Pastebinner::VERSION).not_to be nil
|
expect(Pastebinner::VERSION).not_to be nil
|
||||||
end
|
end
|
||||||
|
|
||||||
it "does something useful" do
|
it 'does something useful' do
|
||||||
expect(false).to eq(true)
|
expect(false).to eq(true)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
require "bundler/setup"
|
require 'bundler/setup'
|
||||||
require "pastebinner"
|
require 'pastebinner'
|
||||||
|
|
||||||
RSpec.configure do |config|
|
RSpec.configure do |config|
|
||||||
# Enable flags like --only-failures and --next-failure
|
# Enable flags like --only-failures and --next-failure
|
||||||
config.example_status_persistence_file_path = ".rspec_status"
|
config.example_status_persistence_file_path = '.rspec_status'
|
||||||
|
|
||||||
# Disable RSpec exposing methods globally on `Module` and `main`
|
# Disable RSpec exposing methods globally on `Module` and `main`
|
||||||
config.disable_monkey_patching!
|
config.disable_monkey_patching!
|
||||||
|
|
Loading…
Add table
Reference in a new issue