require 'rest-client' require 'zlib' require 'json' require 'active_support/all' #require '../modules/nvd_tools' module NvdTools class NvdDownloader attr_accessor :version, :base_url, :base_filename, :years, :filenames_json, :filenames_meta, :client CURRENT_TIME = Time.now MIN_YEAR = '2002' MAX_YEAR = '2019' AVAILABLE_YEARS = (MIN_YEAR..MAX_YEAR).to_a def initialize(client: NvdClient.new) @base_filename = "nvdcve-#{client.version}-" @years = self.years @filenames_json = self.filenames('json.gz') @filenames_meta = self.filenames('meta') @client = client end def years year = NvdDownloader::AVAILABLE_YEARS.map do |year| [year.to_i, year] end.to_h end def filenames(extension) year_filenames = years.map do |k, year| "#{base_filename}#{year}.#{extension}" end other_filenames = [ "#{base_filename}recent.#{extension}", "#{base_filename}modified.#{extension}" ] year_filenames + other_filenames end def read_gzip_stream(gzip_stream) io_stream = StringIO.new(gzip_stream) gz = Zlib::GzipReader.new(io_stream) gz.read end def write_to_file(parsed_json, file_path) File.write(file_path, parsed_json) end def parse_json(json_string) JSON.parse(json_string) end def one_time_import(to_file=false) # experimenting with how i want to do this. # right now its a loop through the filenames # and parse all into a json string and store in an array self.filenames_json.map do |filename| r = client.get(filename) json_string = read_gzip_stream(r.body) parsed_json = parse_json(json_string) { :filename => filename, :json => parsed_json } if to_file filepath = "../data/cve/#{filename}" write_to_file(r.body, filepath) end end # this should be a method that does a one-time import # of all of the json.gz from each year + recent + modified json feeds end def get_metadata_file(filename) file = filenames('meta').select do |name| name == filename end.first r = client.get(file) r.body.split("\r\n") # this should be a method that builds # the modified filename with the meta file extension included end def extract_metadata(metadata) metaf = get_metadata_file(metadata) keys_and_values_str = metaf.map do |line| line.split(":", 2) end json = Hash[keys_and_values_str].to_json JSON.parse(json, { symbolize_names: true }) end def check_metadata(metadata_file_to_check, metadata_to_check_against) # check each k/v pair against the file on disk # return a new hash with the same k as before, but the value being a boolean true or false if the value from the k/v pair end def detect_changes(metadata) # this should be a method that detects changes in the metadata. # run the check_metadata method against the current metadata. # if there is a change, return true, if not return false end end end