bpmcdevitt/pastebinner
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

good additions, got nested object working by loading paste_metadata as an array of hash instead of a string

Browse source
This commit is contained in:
booboy 2019-02-03 02:09:51 -06:00
parent e43e63b98a
commit e104b0b015
4 changed files with 131 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/console
View file

@ -8,5 +8,7 @@ require '../lib/elastic_search_helper'
# (If you use this, don't forget to add pry to your Gemfile!)
require 'pry'
# set restclient logging and setup a pb object
RestClient.log ='stdout'
pb = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
binding.pry

39
lib/elastic_search_helper.rb
View file

@ -9,15 +9,6 @@ class ElasticSearchHelper
@pastebinner = Pastebinner.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
end
def create_index_with_mappings
header = { 'Content-type': 'application/json' }
response = RestClient::Request.execute(
method: :put,
url: "#{server_uri}/#{index}",
headers: header,
payload: self.mappings.to_json)
end
def delete_index
response = RestClient::Request.execute(
method: :delete,
@ -27,35 +18,7 @@ class ElasticSearchHelper
def get_mappings
response = RestClient::Request.execute(
method: :get,
url: "#{server_uri}/#{index}/_mappings")
end
def mappings
{
"mappings": {
"paste": {
"properties": {
"paste_metadata": {
"type": "nested",
"properties": {
"scrape_url": { "type": "text" },
"full_url": { "type": "text" },
"date": { "type": "text" },
"key": { "type": "text" },
"size": { "type": "text" },
"expire": { "type": "text" },
"title": { "type": "text" },
"syntax": { "type": "text" },
"user": { "type": "text" },
"hits": { "type": "integer" }
}
},
"paste_text": { "type": "text" }
}
}
}
}
url: "#{server_uri}/#{index}/_mappings?pretty")
end
def update_mapping(mapping_json)

109
lib/mappings.json
View file

@ -1,12 +1,113 @@
{
{
"pastes": {
"mappings": {
"_doc": {
"paste": {
"properties": {
"paste_metadata": {
"type": "nested"
"properties": {
"date": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"expire": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"full_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"key": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"scrape_url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"size": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"syntax": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"paste_text": { "type": "text" }
"paste_text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}

35
lib/pastebinner.rb
View file

@ -4,6 +4,7 @@
# official docs from pastebin on their api can be found at https://pastebin.com/api
require 'rest-client'
require 'json'
require 'yaml'
class Pastebinner
attr_accessor :api_dev_key, :username, :password
@ -124,23 +125,23 @@ class Pastebinner
method: :get,
url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}"
)
response
YAML.load(response.body)
end
##### PREPARING THE PASTES FOR SERIALIZATION FOR ES CONFORMING TO PER INDEX SEARCHING
##### SEE - https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#_custom_type_field
def hash_pastes(keys)
keys.map do |key|
raw_paste = self.raw_paste_data(key).body
raw_paste_metadata = self.raw_paste_metadata(key).body
if keys.is_a? String
raw_paste = self.raw_paste_data(keys)
raw_paste_metadata = self.raw_paste_metadata(keys)
hash = self.hash_paste(raw_paste, raw_paste_metadata)
end
end
def json_pastes(keys)
self.hash_pastes(keys).map do |paste_hash|
paste_hash.to_json
else
keys.map do |key|
raw_paste = self.raw_paste_data(key).body
raw_paste_metadata = self.raw_paste_metadata(key)
hash = self.hash_paste(raw_paste, raw_paste_metadata)
end
end
end
@ -149,8 +150,18 @@ class Pastebinner
"paste_text": raw_paste_data }
end
def json_paste(raw_paste_data, raw_paste_metadata)
self.hash_paste(raw_paste_data, raw_paste_metadata).to_json
def json_paste(key=nil, keys)
# if we give keys, create an array of 50 json pastes
if keys
self.hash_pastes(keys).map do |paste_hash|
paste_hash.to_json
end
else
# otherwise, just make a json of the 1 raw_paste_data & raw_paste_metadata
raw_paste_data = self.raw_paste_data(key)
raw_paste_metadata = self.raw_paste_metadata(key)
self.hash_paste(raw_paste_data, raw_paste_metadata).to_json
end
end
# keep this method private so we are not letting anyone run any method in our program