bpmcdevitt/data_importer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make poc in github importer use upsert_all

Browse source
This commit is contained in:
Brendan McDevitt 2022-04-22 01:48:49 -05:00
parent ddecc2820c
commit 8261a53800
1 changed files with 22 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
lib/importers/poc_in_github_importer.rb
View file

@ -48,26 +48,26 @@ class PocInGithubImporter < GithubRepo
end
def cve_attrs_from_item(json, cve_attrs = {})
cve_attrs[:github_poc_id] = json['id']
cve_attrs[:name] = json['name']
cve_attrs[:full_name] = json['full_name']
cve_attrs[:owner] = json['owner']
cve_attrs[:html_url] = json['html_url']
cve_attrs[:description] = json['description']
cve_attrs[:fork] = json['fork']
cve_attrs[:created_at] = json['created_at']
cve_attrs[:updated_at] = json['updated_at']
cve_attrs[:pushed_at] = json['pushed_at']
cve_attrs[:stargazers_count] = json['stargazers_count']
cve_attrs[:watchers_count] = json['watchers_count']
cve_attrs[:forks_count] = json['forks_count']
cve_attrs[:allow_forking] = json['allow_forking']
cve_attrs[:is_template] = json['is_template']
cve_attrs[:topics] = json['topics']
cve_attrs[:visibility] = json['visibility']
cve_attrs[:forks] = json['forks']
cve_attrs[:watchers] = json['watchers']
cve_attrs[:score] = json['score']
cve_attrs[:github_poc_id] = json[:id]
cve_attrs[:name] = json[:name]
cve_attrs[:full_name] = json[:full_name]
cve_attrs[:owner] = json[:owner]
cve_attrs[:html_url] = json[:html_url]
cve_attrs[:description] = json[:description]
cve_attrs[:fork] = json[:fork]
cve_attrs[:created_at] = json[:created_at]
cve_attrs[:updated_at] = json[:updated_at]
cve_attrs[:pushed_at] = json[:pushed_at]
cve_attrs[:stargazers_count] = json[:stargazers_count]
cve_attrs[:watchers_count] = json[:watchers_count]
cve_attrs[:forks_count] = json[:forks_count]
cve_attrs[:allow_forking] = json[:allow_forking]
cve_attrs[:is_template] = json[:is_template]
cve_attrs[:topics] = json[:topics]
cve_attrs[:visibility] = json[:visibility]
cve_attrs[:forks] = json[:forks]
cve_attrs[:watchers] = json[:watchers]
cve_attrs[:score] = json[:score]
cve_attrs
end
@ -92,21 +92,8 @@ class PocInGithubImporter < GithubRepo
(1999..Date.today.year).map do |year|
cves_from_json = cves_for_year(year)
ids = cves_from_json.map { |cve| cve[:github_poc_id] }
ids_in_db = GithubPoc.where(github_poc_id: ids).pluck(:github_poc_id)
new_ids = ids - ids_in_db
new_cves = cves_from_json.select { |cve| cve if new_ids.include?(cve[:github_poc_id]) }
puts "Importing any new CVEs from #{year}"
bulk_insert(new_cves)
GithubPoc.upsert_all(cves_from_json, unique_by: :github_poc_id)
end
end
def bulk_insert(cves)
GithubPoc.bulk_insert do |worker|
cves.each do |attrs|
worker.add(attrs)
end
end
end
end
end