misc_rbtools/classes/nvd_downloader.rb

require 'rest-client'
require 'zlib'
require 'json'
require 'active_support/all'
#require '../modules/nvd_tools'

module NvdTools
  class NvdDownloader
    attr_accessor :version, :base_url, :base_filename, :years, :filenames_json, :filenames_meta, :client
   
    CURRENT_TIME = Time.now 
    MIN_YEAR = '2002'
    MAX_YEAR = '2019'
    AVAILABLE_YEARS = (MIN_YEAR..MAX_YEAR).to_a
    
    def initialize(client: NvdClient.new)

      @base_filename = "nvdcve-#{client.version}-"
      @years = self.years
      @filenames_json = self.filenames('json.gz')
      @filenames_meta = self.filenames('meta')
      @client = client
    end
    
    def years
      year = NvdDownloader::AVAILABLE_YEARS.map do |year|
        [year.to_i, year]
      end.to_h
    end
    
    def filenames(extension)
      year_filenames = years.map do |k, year|
        "#{base_filename}#{year}.#{extension}"
      end
      
      other_filenames = [ 
        "#{base_filename}recent.#{extension}",
        "#{base_filename}modified.#{extension}"
      ]
      
      year_filenames + other_filenames
    end
    
    def read_gzip_stream(gzip_stream)
      io_stream = StringIO.new(gzip_stream)
      gz = Zlib::GzipReader.new(io_stream)
      gz.read
    end
    
    def write_to_file(parsed_json, file_path)
      File.write(file_path, parsed_json)
    end
    
    def parse_json(json_string)
      JSON.parse(json_string)
    end
    
    def one_time_import(to_file=false)
      # experimenting with how i want to do this. 
      # right now its a loop through the filenames
      # and parse all into a json string and store in an array
      self.filenames_json.map do |filename|
        r = client.get(filename)
        json_string = read_gzip_stream(r.body)
        parsed_json = parse_json(json_string)

        { :filename => filename,
          :json => parsed_json
        }
        
        if to_file 
        filepath = "../data/cve/#{filename}"
          write_to_file(r.body, filepath)
        end
      end
      # this should be a method that does a one-time import
      # of all of the json.gz from each year + recent + modified json feeds
    end
    
    def get_metadata_file(filename)
      file = filenames('meta').select do |name|
        name == filename
      end.first

      r = client.get(file)
      r.body.split("\r\n")
      # this should be a method that builds 
      # the modified filename with the meta file extension included
    end

    def extract_metadata(metadata)
      metaf = get_metadata_file(metadata)
      keys_and_values_str = metaf.map do |line|
        line.split(":", 2)
      end
      json = Hash[keys_and_values_str].to_json
      JSON.parse(json, { symbolize_names: true })
    end

    def check_metadata(metadata_file_to_check, metadata_to_check_against)
      # check each k/v pair against the file on disk
      # return a new hash with the same k as before, but the value being a boolean true or false if the value from the k/v pair
     
    end

    def detect_changes(metadata)
      # this should be a method that detects changes in the metadata. 
      # run the check_metadata method against the current metadata.
      # if there is a change, return true, if not return false
    end
  end
end