diff --git a/lib/importers/github_advisory_importer.rb b/lib/importers/github_advisory_importer.rb index 8db565a..faab22b 100644 --- a/lib/importers/github_advisory_importer.rb +++ b/lib/importers/github_advisory_importer.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require '/data_importer/lib/importers/github_repo' +require '/data_importer/lib/json_helper' class GithubAdvisoryImporter < GithubRepo # repo has years that begin with 2017 as first GHSA @@ -37,43 +38,25 @@ class GithubAdvisoryImporter < GithubRepo fp_hash = list_jsons_for_year(year) fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons] jsons = fns.map do |fn| - read_json(fn) + json = read_json(fn) + attrs_from_item(json) end jsons.flatten end def attrs_from_item(json) - attrs = {} - attrs[:schema_version] = json['schema_version'] - attrs[:ghsa_id] = json['id'] - attrs[:modified] = json['modified'] - attrs[:published] = json['published'] - attrs[:aliases] = json['aliases'] - attrs[:summary] = json['summary'] - attrs[:details] = json['details'] - attrs[:severity] = json['severity'] - attrs[:affected] = json['affected'] - attrs[:references] = json['references'] - attrs[:database_specific] = json['database_specific'] + attrs = JsonHelper.deep_transform_keys(json) + attrs[:ghsa_id] = json[:id] attrs end - def bulk_insert(jsons) - GithubAdvisory.bulk_insert do |worker| - jsons.each do |json| - attrs = attrs_from_item(json) - worker.add(attrs) - end - end - end - def import pull_or_clone puts 'Now importing GithubAdvisories.' YEAR_RANGE.each do |year| puts "Importing advisory data from #{year}" jsons = read_jsons_for_year(year) - bulk_insert(jsons) + GithubAdvisory.upsert_all(jsons, unique_by: :ghsa_id) end end end