add more upsert_all

lib/importers/cpe_importer.rb

@@ -86,11 +86,13 @@ class CpeImporter
       cpe_attrs_from_item(item)
     end
 
-    Cpe.bulk_insert do |worker|
-      cpes.each do |attrs|
-        worker.add(attrs)
-      end
-    end
+    Cpe.upsert_all(cpes)
+
+    #Cpe.bulk_insert do |worker|
+    #  cpes.each do |attrs|
+    #    worker.add(attrs)
+    #  end
+    #end
   end
 
   def self.cpe_attrs_from_item(item)

lib/importers/github_user_importer.rb

@@ -24,26 +24,17 @@ class GithubUserImporter
 
   def user_h_to_attr(user_h)
     attrs = {}
-    attrs[:github_id] = user_h['id']
-    attrs[:login] = user_h['login']
-    attrs[:name] = user_h['name']
-    attrs[:avatar_url] = user_h['avatarUrl']
-    attrs[:bio] = user_h['bio']
-    attrs[:bio_html] = user_h['bioHTML']
-    attrs[:location] = user_h['location']
-    attrs[:repositories] = user_h['repositories']
+    attrs[:github_id] = user_h.dig('id')
+    attrs[:login] = user_h.dig('login')
+    attrs[:name] = user_h.dig('name')
+    attrs[:avatar_url] = user_h.dig('avatarUrl')
+    attrs[:bio] = user_h.dig('bio')
+    attrs[:bio_html] = user_h.dig('bioHTML')
+    attrs[:location] = user_h.dig('location')
+    attrs[:repositories] = user_h.dig('repositories')
     attrs
   end
 
-  def bulk_insert(username_hashes)
-    GithubUser.bulk_insert do |worker|
-      username_hashes.each do |username_hash|
-        attrs = user_h_to_attr(username_hash)
-        worker.add(attrs)
-      end
-    end
-  end
-
   def import
     if filepath.nil?
       puts 'Please provide a filepath in the projects data dir named github_usernames.txt with one username per line.'
@@ -51,7 +42,8 @@ class GithubUserImporter
       puts 'Now importing GithubUsers'
       usernames = username_hashes.map { |h| h['login'] }
       puts "Now importing data from the following usernames: #{usernames}"
-      bulk_insert(username_hashes)
+      username_attrs = username_hashes.map { |user_h| user_h_to_attr(user_h) }
+      GithubUser.upsert_all(username_attrs)
     end
   end
 end

lib/importers/inthewild_cve_exploit_importer.rb

@@ -30,14 +30,6 @@ class InthewildCveExploitImporter
     cve_attrs
   end
 
-  def bulk_insert(cves)
-    InthewildCveExploit.bulk_insert do |worker|
-      cves.each do |attrs|
-        worker.add(attrs)
-      end
-    end
-  end
-
   def import
     feed = get_exploit_feed
     puts 'Now importing InthewildCveExploits.'
@@ -45,6 +37,6 @@ class InthewildCveExploitImporter
       cve_attrs_from_item(cve_entry)
     end
 
-    bulk_insert(cves)
+    InthewildCveExploit.upsert_all(cves)
   end
 end