added gsd importer

app/controllers/gsds_controller.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class GsdsController < ApplicationController
+  def index
+    @gsds = Gsd.all
+    render json: @gsds.to_json
+  end
+
+  def show
+    @gsd = Gsd.find_by_id(params[:gsd_id])
+    render json: @gsd.to_json
+  end
+
+  def show_year
+    @gsds_for_year = Gsd.from_year(params[:year])
+    render json: @gsds_for_year.to_json
+  end
+end

app/models/gsd.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Gsd < ActiveRecord::Base
+  def self.find_by_cve_id(id)
+    find_by(cve_id: id)
+  end
+
+  def self.from_year(year)
+    where('cve_id LIKE ?', "CVE-#{year}-%")
+  end
+end

app/views/gsds/index.html.erb

@@ -0,0 +1 @@
+<h1>Gsds#index</h1>

app/views/gsds/show.html.erb

@@ -0,0 +1 @@
+

app/views/gsds/show_years.html.erb

@@ -0,0 +1,2 @@
+<h1> params[:year] </h1>
+<h1> <%= @gsds_for_year %> </h1>

app/workers/gsd_importer_worker.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require '/data_importer/lib/importers/gsd_importer'
+
+class GsdImporterWorker
+  include Faktory::Job
+
+  def perform(*args)
+    puts "Hello, I am #{jid} with args #{args}"
+    GsdImporter.new.import
+  end
+end

crontab.yaml

@@ -53,4 +53,10 @@ jobs:
     schedule: "@hourly"
     retries: 1
     queue: default
+    priority: 5
+  - job: GsdImporterWorker
+    args: []
+    schedule: "@every 4h00m00s"
+    retries: 1
+    queue: default
     priority: 5

db/migrate/20220419203353_create_gsds.rb

@@ -0,0 +1,12 @@
+class CreateGsds < ActiveRecord::Migration[7.0]
+  def change
+    create_table :gsds do |t|
+      t.string :gsd_id
+      t.index :gsd_id, unique: true
+      t.string :cve_id
+      t.jsonb :gsd
+      t.jsonb :namespaces
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_19_203353) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -118,6 +118,16 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
     t.jsonb "repositories"
   end
 
+  create_table "gsds", force: :cascade do |t|
+    t.string "gsd_id"
+    t.string "cve_id"
+    t.jsonb "gsd"
+    t.jsonb "namespaces"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["gsd_id"], name: "index_gsds_on_gsd_id", unique: true
+  end
+
   create_table "inthewild_cve_exploits", force: :cascade do |t|
     t.string "cve_id"
     t.string "earliest_report"

db/seeds.rb

@@ -17,6 +17,7 @@ require '/data_importer/lib/importers/cvemon_cve_importer'
 require '/data_importer/lib/importers/cna_importer'
 require '/data_importer/lib/importers/github_advisory_importer'
 require '/data_importer/lib/importers/github_user_importer'
+require '/data_importer/lib/importers/gsd_importer.rb'
 
 def line_sep
   puts '----------' * 12
@@ -24,6 +25,7 @@ end
 
 def perform
   import_cves
+  import_gsds
   import_github_pocs
   import_trickest_poc_cves
   import_inthewild_cve_exploits
@@ -44,6 +46,11 @@ def import_cpes
   CpeImporter.download_and_import
 end
 
+def import_gsds
+    line_sep
+    GsdImporter.new.import
+end
+
 def import_github_pocs
   line_sep
   PocInGithubImporter.new.import

lib/importers/gsd_importer.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'git'
+require 'json'
+require 'date'
+require '/data_importer/lib/importers/github_repo'
+require '/data_importer/lib/json_helper'
+
+class GsdImporter < GithubRepo
+  EXPECTED_KEYS = %i[
+    cve_id
+    gsd_id
+    gsd
+    namespaces
+  ].freeze
+
+  EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze
+
+  def initialize
+    super(repo_url = 'https://github.com/cloudsecurityalliance/gsd-database.git', repo_path = '/data_importer/data/gsd_database')
+  end
+
+  def list_jsons_for_year(year)
+    year_fp = "#{repo_path}/#{year}/*"
+    Dir["#{year_fp}/*.json"]
+  end
+
+  def read_jsons_for_year(year)
+    filenames = list_jsons_for_year(year)
+    hashes = filenames.map do |filename|
+      json = read_json(filename)
+      json_transformed = JsonHelper.deep_transform_keys(json)
+      append_ids_to_hash(json_transformed)
+    end
+    hashes.map { |h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) }
+  end
+
+  def append_ids_to_hash(json)
+    json[:cve_id] = json.dig(:gsd, :alias)
+    json[:gsd_id] = json.dig(:gsd, :id)
+    json
+  end
+
+  # they like to post descriptions with \u0000 and it doesnt make postgres happy
+  def sanitize_gitlab_advisories(json)
+    if json[:namespaces].keys.include? (:"gitlab.com")
+      advisories = json[:namespaces][:"gitlab.com"][:advisories]
+      sanitized_advisories = advisories.map do |advisory|
+        description = advisory[:description]
+        sanitized_description = JsonHelper.fix_null_byte(description)
+        advisory[:description] = sanitized_description
+        advisory
+      end
+      json[:namespaces][:"gitlab.com"][:advisories] = sanitized_advisories
+    end
+    json
+  end
+
+  def import
+    pull_or_clone
+    puts "Now starting import for #{repo_url}."
+    puts '----------' * 12
+    (1999..Date.today.year).map do |year|
+      puts "Now importing GSDs for #{year}"
+      gsds = read_jsons_for_year(year)
+      gsds.each do |h|
+        # fixes a suse cve from 2009 that didnt have any namespaces set in the json.
+        if h[:namespaces].nil?
+          Gsd.upsert_all([h], unique_by: :gsd_id)
+        else 
+          sanitized_h = sanitize_gitlab_advisories(h)
+          Gsd.upsert_all([sanitized_h], unique_by: :gsd_id)
+        end
+      end
+    end
+  end
+end

lib/json_helper.rb

@@ -12,4 +12,9 @@ class JsonHelper
   def self.symbolize_names_snake_case(json_hash)
     json_hash.deep_transform_keys { |k| k.to_s.underscore.to_sym }
   end
+
+  def self.fix_null_byte(s)
+    s.gsub("`\u0000`", "null_byte")
+  end
+
 end