class EsSearchHelper attr_accessor :server_uri, :index, :api_client, :doctype, :redis_connection DEFAULT_METHOD = :post def initialize(server_uri, index, doctype='paste') @server_uri = server_uri @index = index @doctype = doctype @api_client = ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password']) @redis_connection = RedisHelper.new(host: ENV['REDIS_SERVER_URL_PASTES_HOSTS'], db: ENV['REDIS_SERVER_URL_PASTES_DB']) end def delete_index response = RestClient::Request.execute( method: :delete, url: "#{server_uri}/#{index}") end def get_mappings response = RestClient::Request.execute( method: :get, url: "#{server_uri}/#{index}/_mappings?pretty") end def create_mapping(mapping_json) header = { 'Content-type': 'application/json' } response = RestClient::Request.execute( method: :put, url: "#{server_uri}/#{index}", headers: header, payload: mapping_json) end def update_mapping(mapping_json) header = { 'Content-type': 'application/json' } response = RestClient::Request.execute( method: :put, url: "#{server_uri}/#{index}/_mapping/#{doctype}", payload: mapping_json, headers: header) end def json_to_es(paste_json, method=nil) header = { 'Content-type': 'application/json' } response = RestClient::Request.execute( method: method ||= DEFAULT_METHOD, url: "#{server_uri}/#{index}/#{doctype}", headers: header, payload: paste_json) end def json_to_es_bulk(array_of_paste_json) array_of_paste_json.each do |paste_json| self.json_to_es(paste_json) end end def send_jsons_to_es(paste_max) pastes = api_client.scrape_public_pastes(paste_max) keys = api_client.get_unique_paste_keys(pastes) keys_for_request = check_and_send_keys_redis(keys) json_data = api_client.json_paste(keys_for_request) self.json_to_es_bulk(json_data) end def check_and_send_keys_redis(keys) keys_in_redis = keys.select do |key| key if redis_connection.paste_key_exists?(key) end # we dont want to send a request if we already have the key present in redis. keys_for_request = keys - keys_in_redis keys_for_request.map do |key| redis_connection.send_paste_key(key) end return keys_for_request end end