From 8261a53800e54fe762d2608789014a91a0a586ba Mon Sep 17 00:00:00 2001
From: Brendan McDevitt <brendan@mcdevitt.tech>
Date: Fri, 22 Apr 2022 01:48:49 -0500
Subject: [PATCH] make poc in github importer use upsert_all

---
 lib/importers/poc_in_github_importer.rb | 57 ++++++++++---------------
 1 file changed, 22 insertions(+), 35 deletions(-)

diff --git a/lib/importers/poc_in_github_importer.rb b/lib/importers/poc_in_github_importer.rb
index 82c61db..0d54ca6 100644
--- a/lib/importers/poc_in_github_importer.rb
+++ b/lib/importers/poc_in_github_importer.rb
@@ -48,26 +48,26 @@ class PocInGithubImporter < GithubRepo
   end
 
   def cve_attrs_from_item(json, cve_attrs = {})
-    cve_attrs[:github_poc_id] = json['id']
-    cve_attrs[:name] = json['name']
-    cve_attrs[:full_name] = json['full_name']
-    cve_attrs[:owner] = json['owner']
-    cve_attrs[:html_url] = json['html_url']
-    cve_attrs[:description] = json['description']
-    cve_attrs[:fork] = json['fork']
-    cve_attrs[:created_at] = json['created_at']
-    cve_attrs[:updated_at] = json['updated_at']
-    cve_attrs[:pushed_at] = json['pushed_at']
-    cve_attrs[:stargazers_count] = json['stargazers_count']
-    cve_attrs[:watchers_count] = json['watchers_count']
-    cve_attrs[:forks_count] = json['forks_count']
-    cve_attrs[:allow_forking] = json['allow_forking']
-    cve_attrs[:is_template] = json['is_template']
-    cve_attrs[:topics] = json['topics']
-    cve_attrs[:visibility] = json['visibility']
-    cve_attrs[:forks] = json['forks']
-    cve_attrs[:watchers] = json['watchers']
-    cve_attrs[:score] = json['score']
+    cve_attrs[:github_poc_id] = json[:id]
+    cve_attrs[:name] = json[:name]
+    cve_attrs[:full_name] = json[:full_name]
+    cve_attrs[:owner] = json[:owner]
+    cve_attrs[:html_url] = json[:html_url]
+    cve_attrs[:description] = json[:description]
+    cve_attrs[:fork] = json[:fork]
+    cve_attrs[:created_at] = json[:created_at]
+    cve_attrs[:updated_at] = json[:updated_at]
+    cve_attrs[:pushed_at] = json[:pushed_at]
+    cve_attrs[:stargazers_count] = json[:stargazers_count]
+    cve_attrs[:watchers_count] = json[:watchers_count]
+    cve_attrs[:forks_count] = json[:forks_count]
+    cve_attrs[:allow_forking] = json[:allow_forking]
+    cve_attrs[:is_template] = json[:is_template]
+    cve_attrs[:topics] = json[:topics]
+    cve_attrs[:visibility] = json[:visibility]
+    cve_attrs[:forks] = json[:forks]
+    cve_attrs[:watchers] = json[:watchers]
+    cve_attrs[:score] = json[:score]
     cve_attrs
   end
 
@@ -92,21 +92,8 @@ class PocInGithubImporter < GithubRepo
     (1999..Date.today.year).map do |year|
       cves_from_json = cves_for_year(year)
 
-      ids = cves_from_json.map { |cve| cve[:github_poc_id] }
-      ids_in_db = GithubPoc.where(github_poc_id: ids).pluck(:github_poc_id)
-
-      new_ids = ids - ids_in_db
-      new_cves = cves_from_json.select { |cve| cve if new_ids.include?(cve[:github_poc_id]) }
       puts "Importing any new CVEs from #{year}"
-      bulk_insert(new_cves)
+      GithubPoc.upsert_all(cves_from_json, unique_by: :github_poc_id)
     end
   end
-
-  def bulk_insert(cves)
-    GithubPoc.bulk_insert do |worker|
-      cves.each do |attrs|
-        worker.add(attrs)
-      end
-    end
-  end
-end
+end
\ No newline at end of file