From ef3a454a26dfdc2d9d3612ca4f4364055c7bbde7 Mon Sep 17 00:00:00 2001
From: Brendan McDevitt <brendan@mcdevitt.tech>
Date: Tue, 19 Apr 2022 14:50:18 -0500
Subject: [PATCH] add timestamps to github advisories so upsert_all works the
 best

---
 ...20220411181501_create_github_advisories.rb |  1 +
 db/schema.rb                                  |  2 ++
 lib/importers/github_advisory_importer.rb     | 19 ++++++++++++++++++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/db/migrate/20220411181501_create_github_advisories.rb b/db/migrate/20220411181501_create_github_advisories.rb
index ffb8b08..1ed2b1f 100644
--- a/db/migrate/20220411181501_create_github_advisories.rb
+++ b/db/migrate/20220411181501_create_github_advisories.rb
@@ -15,6 +15,7 @@ class CreateGithubAdvisories < ActiveRecord::Migration[7.0]
       t.jsonb :affected
       t.jsonb :references
       t.jsonb :database_specific
+      t.timestamps
     end
   end
 end
diff --git a/db/schema.rb b/db/schema.rb
index 0c544d3..2f5d988 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -77,6 +77,8 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
     t.jsonb "affected"
     t.jsonb "references"
     t.jsonb "database_specific"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
     t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true
   end
 
diff --git a/lib/importers/github_advisory_importer.rb b/lib/importers/github_advisory_importer.rb
index faab22b..9afb47b 100644
--- a/lib/importers/github_advisory_importer.rb
+++ b/lib/importers/github_advisory_importer.rb
@@ -7,6 +7,22 @@ class GithubAdvisoryImporter < GithubRepo
   # repo has years that begin with 2017 as first GHSA
   YEAR_RANGE = (2017..Date.today.year).freeze
 
+  EXPECTED_KEYS = %i[
+    schema_version
+    ghsa_id
+    modified
+    published
+    aliases
+    summary
+    details
+    severity
+    affected
+    references
+    database_specific
+  ].freeze
+
+  EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze
+
   def initialize
     super(repo_url = 'https://github.com/github/advisory-database.git', repo_path = '/data_importer/data/github_advisories')
   end
@@ -56,7 +72,8 @@ class GithubAdvisoryImporter < GithubRepo
     YEAR_RANGE.each do |year|
       puts "Importing advisory data from #{year}"
       jsons = read_jsons_for_year(year)
-      GithubAdvisory.upsert_all(jsons, unique_by: :ghsa_id)
+      hashes = jsons.map { |h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) }
+      GithubAdvisory.upsert_all(hashes, unique_by: :ghsa_id)
     end
   end
 end