require 'rest-client' require 'zlib' require 'json' require '../modules/nvd_tools' module NvdTools class NvdDownloader attr_accessor :version, :base_url, :base_filename, :years, :filenames_json, :client MIN_YEAR = '2002' MAX_YEAR = '2019' AVAILABLE_YEARS = (MIN_YEAR..MAX_YEAR).to_a GZIPPED_JSON_EXTENSION = 'json.gz' def initialize(client: NvdClient.new) @base_filename = "nvdcve-#{client.version}-" @years = self.years @filenames_json = self.filenames(GZIPPED_JSON_EXTENSION) @client = client end def years year = NvdDownloader::AVAILABLE_YEARS.map do |year| [year.to_i, year] end.to_h end def filenames(extension) year_filenames = years.map do |k, year| "#{base_filename}#{year}.#{extension}" end other_filenames = [ "#{base_filename}recent.#{extension}", "#{base_filename}modified.#{extension}" ] year_filenames + other_filenames end def read_gzip_stream(gzip_stream) io_stream = StringIO.new(gzip_stream) gz = Zlib::GzipReader.new(io_stream) gz.read end def write_to_file(parsed_json, file_path) File.write(file_path, parsed_json) end def parse_json(json_string) JSON.parse(json_string) end def one_time_import # experimenting with how i want to do this. # right now its a loop through the filenames # and parse all into a json string and store in an array self.filenames_json.map do |filename| r = client.get(filename) sleep(1) gzip_stream = r.body json_string = read_gzip_stream(gzip_stream) parse_json(json_string) end # this should be a method that does a one-time import # of all of the json.gz from each year + recent + modified json feeds end def modified_meta # this should be a method that builds # the modified filename with the meta file extension included end def check_metafile(metafile) # open the metafile, build a hash of k/v pairs of the data inside of the file # check each k/v pair against the file on disk # return a new hash with the same k as before, but the value being a boolean true or false if the value from the k/v pair end def detect_changes(metafile) # this should be a method that detects changes in the metafile. # run the check_metafile method against the current metafile on disk. # if there is a change, return true, if not return false end end end