bpmcdevitt/data_importer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make github advisory importer use upsert_all

Browse source
This commit is contained in:
Brendan McDevitt 2022-04-19 14:22:59 -05:00
parent a2fa2fbb62
commit 9154a9cb9c
1 changed files with 6 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
lib/importers/github_advisory_importer.rb
View file

@ -1,6 +1,7 @@
# frozen_string_literal: true
require '/data_importer/lib/importers/github_repo'
require '/data_importer/lib/json_helper'
class GithubAdvisoryImporter < GithubRepo
# repo has years that begin with 2017 as first GHSA
@ -37,43 +38,25 @@ class GithubAdvisoryImporter < GithubRepo
fp_hash = list_jsons_for_year(year)
fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons]
jsons = fns.map do |fn|
read_json(fn)
json = read_json(fn)
attrs_from_item(json)
end
jsons.flatten
end
def attrs_from_item(json)
attrs = {}
attrs[:schema_version] = json['schema_version']
attrs[:ghsa_id] = json['id']
attrs[:modified] = json['modified']
attrs[:published] = json['published']
attrs[:aliases] = json['aliases']
attrs[:summary] = json['summary']
attrs[:details] = json['details']
attrs[:severity] = json['severity']
attrs[:affected] = json['affected']
attrs[:references] = json['references']
attrs[:database_specific] = json['database_specific']
attrs = JsonHelper.deep_transform_keys(json)
attrs[:ghsa_id] = json[:id]
attrs
end
def bulk_insert(jsons)
GithubAdvisory.bulk_insert do |worker|
jsons.each do |json|
attrs = attrs_from_item(json)
worker.add(attrs)
end
end
end
def import
pull_or_clone
puts 'Now importing GithubAdvisories.'
YEAR_RANGE.each do |year|
puts "Importing advisory data from #{year}"
jsons = read_jsons_for_year(year)
bulk_insert(jsons)
GithubAdvisory.upsert_all(jsons, unique_by: :ghsa_id)
end
end
end