pastebinner/lib/pastebinner.rb

#!/usr/bin/env ruby
# author: brendan mcdevitt
# a ruby wrapper around all of the methods pastebin provides with its api
# official docs from pastebin on their api can be found at https://pastebin.com/api
require 'rest-client'
require 'json'

class Pastebinner
 attr_accessor :api_dev_key, :username, :password

    def initialize(api_dev_key, username, password)
      @api_dev_key = api_dev_key
      @username = username
      @password = password
      @base_api_url = 'https://pastebin.com/api'
      @scraping_api_url = 'https://scrape.pastebin.com'
    end

    # this should be a hash of { endpoint_name: '/url_endpoint.php'}
    ENDPOINTS = { :login => '/api_login.php',
                  :post => '/api_post.php',
                  :raw => '/api_raw.php',
                  :scraping => '/api_scraping.php',
                  :scrape_item => '/api_scrape_item.php',
                  :scrape_item_meta => '/api_scrape_item_meta.php' }

    # basic example hash for creating a paste:
    # params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data}

    # required params:
    # api_dev_key - your unique developer api key
    # api_option - set as paste, this will indicate you want to create a new paste
    # api_paste_code - this is the text that will be written inside of your paste

    # optional params:
    # api_user_key - this parameter is part of the login system, which is explained further down the page
    # api_paste_name - this will be the name / title of your paste
    # api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page
    # api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2
    # api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page

    # example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data }
    def create_paste(params)
      execute_query(:api_post, params)
    end

    def api_user_key
      # returns a user session key that can be used as the api_user_key param
      @api_user_key ||= RestClient::Request.execute({
                                                  method: :post,
                                                  url: @base_api_url + ENDPOINTS[:login],
                                                  payload: { 'api_dev_key': @api_dev_key,
                                                             'api_user_name': @username,
                                                             'api_user_password': @password }})
    end

    def list_user_pastes
      params = { 'api_dev_key': api_dev_key,
                 'api_user_key': api_user_key,
                 'api_results_limit': '100',
                 'api_option': 'list'
               }
      execute_query(:api_post, params)
    end

    def list_trending_pastes
      params = { 'api_dev_key': api_dev_key,
                 'api_option': 'trends'
               }
      execute_query(:api_post, params)
    end

    def list_raw_user_paste(api_paste_key)
      params = { 'api_dev_key': api_dev_key,
                 'api_user_key': api_user_key,
                 'api_paste_key': api_paste_key,
                 'api_option': 'show_paste'
               }
      execute_query(:api_post, params)
    end

    # api_paste_key = this is the unique key of the paste data you want to delete.
    def delete_user_paste(api_paste_key)
      params = { 'api_dev_key': api_dev_key,
                 'api_user_key': api_user_key,
                 'api_paste_key': api_paste_key,
                 'api_option': 'delete'
               }
      execute_query(:api_post, params)
    end

    def get_user_info
      params = { 'api_dev_key': api_dev_key,
               }
    end

    def api_post(params)
      response = RestClient::Request.execute(
        method: :post,
        url: @base_api_url + ENDPOINTS[:post],
        payload: params)
    end

    # params is optional for now. to query specific language ?lang=ruby as an example
    def scrape_public_pastes(params = nil)
      response = RestClient::Request.execute(
        method: :get,
        url: @scraping_api_url + ENDPOINTS[:scraping])
    end

    # will extract just the keys from recent public pastes
    def get_unique_paste_keys(public_pastes)
      pp = JSON.parse(public_pastes)
      pp.map {|p| p['key']}
    end

    def raw_paste_data(unique_paste_key)
      response = RestClient::Request.execute(
        method: :get,
        url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}")
    end

    def raw_paste_metadata(unique_paste_key)
      response = RestClient::Request.execute(
        method: :get,
        url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}")
      response
    end
   
    ##### PREPARING THE PASTES FOR SERIALIZATION FOR ES CONFORMING TO PER INDEX SEARCHING
    ##### SEE - https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#_custom_type_field
 
    def http_to_es(es_uri, payload, method)
      header = { 'Content-type': 'application/json' }
      response = RestClient::Request.execute(
        method: method,
        url: es_uri, 
        headers: header,
        payload: payload)
    end

    def create_index_es(es_uri, index)
      response = RestClient::Request.execute(
        method: :put,
        url: es_uri + index)
    end

    def hash_pastes(keys)
      keys.map do |key|
        raw_paste = self.raw_paste_data(key).body
        raw_paste_metadata = self.raw_paste_metadata(key).body
        hash = self.hash_paste(raw_paste, raw_paste_metadata)
      end
    end
     
    def hash_paste(raw_paste_data, raw_paste_metadata)
      { "paste_metadata": raw_paste_metadata,
        "paste_text": raw_paste_data }
    end
    
    def send_es_bulk(esi_uri, json_data)
      method = :post
      json_data.each do |payload|
        self.http_to_es(esi_uri, payload, method)
      end
    end

    def data_mappings
    # metadata mappings
    # send a PUT
      {
        "mappings": {
          "_doc": {
            "properties": {
              "type": { "type": "keyword" },
              "paste_metadata": { "type": "nested" },
              "properties": [ {
                "scrape_url": { "type": "string" },
                "full_url": { "type": "string" },
                "date": { "type": "string" },
                "size": { "type": "string" },
                "expire": { "type": "string" },
                "title": { "type": "string" },
                "syntax": { "type": "string" },
                "user": { "type": "string" },
                "hits": { "type": "string" }
              } ],
              "paste_text": { "type": "string" }
            }
          }
        }
      }
    end

    # keep this method private so we are not letting anyone run any method in our program
    private
    # this will be the main way to execute any of these methods. this has the exception handling taken care of.
    def execute_query(selector, *args)
      begin
        send(selector, *args)
      rescue RestClient::ExceptionWithResponse => e
        puts e.message
      end
    end
    # make my own exception class
    # inherit ruby standard error class
end