80 lines
2.3 KiB
Ruby
80 lines
2.3 KiB
Ruby
|
class EsSearchHelper
|
||
|
|
attr_accessor :server_uri, :index, :api_client, :doctype, :redis_connection
|
||
|
|
DEFAULT_METHOD = :post
|
||
|
|
|
||
|
|
def initialize(server_uri, index, doctype='paste')
|
||
|
|
@server_uri = server_uri
|
||
|
|
@index = index
|
||
|
|
@doctype = doctype
|
||
|
|
@api_client = ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||
|
|
@redis_connection = RedisHelper.new(host: ENV['REDIS_SERVER_URL_PASTES_HOSTS'], db: ENV['REDIS_SERVER_URL_PASTES_DB'])
|
||
|
|
end
|
||
|
|
|
||
|
|
def delete_index
|
||
|
|
response = RestClient::Request.execute(
|
||
|
|
method: :delete,
|
||
|
|
url: "#{server_uri}/#{index}")
|
||
|
|
end
|
||
|
|
|
||
|
|
def get_mappings
|
||
|
|
response = RestClient::Request.execute(
|
||
|
|
method: :get,
|
||
|
|
url: "#{server_uri}/#{index}/_mappings?pretty")
|
||
|
|
end
|
||
|
|
|
||
|
|
def create_mapping(mapping_json)
|
||
|
|
header = { 'Content-type': 'application/json' }
|
||
|
|
response = RestClient::Request.execute(
|
||
|
|
method: :put,
|
||
|
|
url: "#{server_uri}/#{index}",
|
||
|
|
headers: header,
|
||
|
|
payload: mapping_json)
|
||
|
|
end
|
||
|
|
|
||
|
|
def update_mapping(mapping_json)
|
||
|
|
header = { 'Content-type': 'application/json' }
|
||
|
|
response = RestClient::Request.execute(
|
||
|
|
method: :put,
|
||
|
|
url: "#{server_uri}/#{index}/_mapping/#{doctype}",
|
||
|
|
payload: mapping_json,
|
||
|
|
headers: header)
|
||
|
|
end
|
||
|
|
|
||
|
|
def json_to_es(paste_json, method=nil)
|
||
|
|
header = { 'Content-type': 'application/json' }
|
||
|
|
response = RestClient::Request.execute(
|
||
|
|
method: method ||= DEFAULT_METHOD,
|
||
|
|
url: "#{server_uri}/#{index}/#{doctype}",
|
||
|
|
headers: header,
|
||
|
|
payload: paste_json)
|
||
|
|
end
|
||
|
|
|
||
|
|
def json_to_es_bulk(array_of_paste_json)
|
||
|
|
array_of_paste_json.each do |paste_json|
|
||
|
|
self.json_to_es(paste_json)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
def send_jsons_to_es(paste_max)
|
||
|
|
pastes = api_client.scrape_public_pastes(paste_max)
|
||
|
|
keys = api_client.get_unique_paste_keys(pastes)
|
||
|
|
keys_for_request = check_and_send_keys_redis(keys)
|
||
|
|
|
||
|
|
json_data = api_client.json_paste(keys_for_request)
|
||
|
|
self.json_to_es_bulk(json_data)
|
||
|
|
end
|
||
|
|
|
||
|
|
def check_and_send_keys_redis(keys)
|
||
|
|
keys_in_redis = keys.select do |key|
|
||
|
|
key if redis_connection.paste_key_exists?(key)
|
||
|
|
end
|
||
|
|
|
||
|
|
# we dont want to send a request if we already have the key present in redis.
|
||
|
|
keys_for_request = keys - keys_in_redis
|
||
|
|
|
||
|
|
keys_for_request.map do |key|
|
||
|
|
redis_connection.send_paste_key(key)
|
||
|
|
end
|
||
|
|
return keys_for_request
|
||
|
|
end
|
||
|
|
end
|