From fe415b0c13548279808fa4f7ec8d1b4230891a06 Mon Sep 17 00:00:00 2001
From: Brendan McDevitt <brendan@mcdevitt.tech>
Date: Mon, 25 Apr 2022 12:58:39 -0500
Subject: [PATCH] add more upsert_all

---
 lib/importers/cpe_importer.rb                 | 12 ++++----
 lib/importers/github_user_importer.rb         | 28 +++++++------------
 .../inthewild_cve_exploit_importer.rb         | 10 +------
 3 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/lib/importers/cpe_importer.rb b/lib/importers/cpe_importer.rb
index 28d3515..8ae19c2 100644
--- a/lib/importers/cpe_importer.rb
+++ b/lib/importers/cpe_importer.rb
@@ -86,11 +86,13 @@ class CpeImporter
       cpe_attrs_from_item(item)
     end
 
-    Cpe.bulk_insert do |worker|
-      cpes.each do |attrs|
-        worker.add(attrs)
-      end
-    end
+    Cpe.upsert_all(cpes)
+
+    #Cpe.bulk_insert do |worker|
+    #  cpes.each do |attrs|
+    #    worker.add(attrs)
+    #  end
+    #end
   end
 
   def self.cpe_attrs_from_item(item)
diff --git a/lib/importers/github_user_importer.rb b/lib/importers/github_user_importer.rb
index 352daad..7cd7506 100644
--- a/lib/importers/github_user_importer.rb
+++ b/lib/importers/github_user_importer.rb
@@ -24,26 +24,17 @@ class GithubUserImporter
 
   def user_h_to_attr(user_h)
     attrs = {}
-    attrs[:github_id] = user_h['id']
-    attrs[:login] = user_h['login']
-    attrs[:name] = user_h['name']
-    attrs[:avatar_url] = user_h['avatarUrl']
-    attrs[:bio] = user_h['bio']
-    attrs[:bio_html] = user_h['bioHTML']
-    attrs[:location] = user_h['location']
-    attrs[:repositories] = user_h['repositories']
+    attrs[:github_id] = user_h.dig('id')
+    attrs[:login] = user_h.dig('login')
+    attrs[:name] = user_h.dig('name')
+    attrs[:avatar_url] = user_h.dig('avatarUrl')
+    attrs[:bio] = user_h.dig('bio')
+    attrs[:bio_html] = user_h.dig('bioHTML')
+    attrs[:location] = user_h.dig('location')
+    attrs[:repositories] = user_h.dig('repositories')
     attrs
   end
 
-  def bulk_insert(username_hashes)
-    GithubUser.bulk_insert do |worker|
-      username_hashes.each do |username_hash|
-        attrs = user_h_to_attr(username_hash)
-        worker.add(attrs)
-      end
-    end
-  end
-
   def import
     if filepath.nil?
       puts 'Please provide a filepath in the projects data dir named github_usernames.txt with one username per line.'
@@ -51,7 +42,8 @@ class GithubUserImporter
       puts 'Now importing GithubUsers'
       usernames = username_hashes.map { |h| h['login'] }
       puts "Now importing data from the following usernames: #{usernames}"
-      bulk_insert(username_hashes)
+      username_attrs = username_hashes.map { |user_h| user_h_to_attr(user_h) }
+      GithubUser.upsert_all(username_attrs)
     end
   end
 end
diff --git a/lib/importers/inthewild_cve_exploit_importer.rb b/lib/importers/inthewild_cve_exploit_importer.rb
index a428daa..14c4f31 100644
--- a/lib/importers/inthewild_cve_exploit_importer.rb
+++ b/lib/importers/inthewild_cve_exploit_importer.rb
@@ -30,14 +30,6 @@ class InthewildCveExploitImporter
     cve_attrs
   end
 
-  def bulk_insert(cves)
-    InthewildCveExploit.bulk_insert do |worker|
-      cves.each do |attrs|
-        worker.add(attrs)
-      end
-    end
-  end
-
   def import
     feed = get_exploit_feed
     puts 'Now importing InthewildCveExploits.'
@@ -45,6 +37,6 @@ class InthewildCveExploitImporter
       cve_attrs_from_item(cve_entry)
     end
 
-    bulk_insert(cves)
+    InthewildCveExploit.upsert_all(cves)
   end
 end