now we pull github security advisories

Gemfile

@@ -14,6 +14,8 @@ gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support
 gem 'nokogiri'
 gem 'graphql'
 gem 'graphql-client'
+gem 'rubocop'
+gem 'rubocop-rails'
 
 # Use postgres as the database for Active Record
 gem 'pg'

Gemfile.lock

@@ -79,6 +79,7 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     archive-zip (0.12.0)
       io-like (~> 0.3.0)
+    ast (2.4.2)
     awesome_print (1.9.2)
     bindex (0.8.1)
     bootsnap (1.11.1)
@@ -215,6 +216,9 @@ GEM
     nokogiri (1.13.3)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
+    parallel (1.22.1)
+    parser (3.1.1.0)
+      ast (~> 2.4.1)
     pg (1.3.5)
     pry (0.14.1)
       coderay (~> 1.1)
@@ -264,6 +268,7 @@ GEM
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
+    rainbow (3.1.1)
     rake (13.0.6)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
@@ -276,6 +281,22 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
+    rubocop (1.27.0)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.16.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.17.0)
+      parser (>= 3.1.1.0)
+    rubocop-rails (2.14.2)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.7.0, < 2.0)
+    ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
     ruby_dep (1.5.0)
     rubyzip (2.3.2)
@@ -331,6 +352,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.1)
+    unicode-display_width (2.1.0)
     web-console (4.2.0)
       actionview (>= 6.0.0)
       activemodel (>= 6.0.0)
@@ -375,6 +397,8 @@ DEPENDENCIES
   rails (~> 7.0.0)
   railties
   rest-client
+  rubocop
+  rubocop-rails
   sass-rails
   selenium-webdriver
   spring

README.md

@@ -3,8 +3,9 @@
 This is a rails/postgres application that will serve json data from the following data sources:
 - Cves
 - Cpes
-- CNA Security Advisories
+- CNA security advisories
-- Github repositories that track public exploits for Cves.
+- GHSA Github security advisories
+- Github repositories that track public exploits for cves.
 
 Check the HTTP API section below for specific endpoints that can be queried via http. 
 
@@ -13,6 +14,7 @@ Check the HTTP API section below for specific endpoints that can be queried via
 - `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format.
 - `Cna` data from [mitre](https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json).
 - `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo.
+- `GithubAdvisories` data from [github_advisories_database](https://github.com/github/advisory-database/) github repo.
 - `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed.
 - `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo.
 - `CvemonCve` data from [ARPSyndicate](https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json) github repo.
@@ -54,6 +56,12 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
   get "/cnas/cna/:cna_id", to: "cnas#show_for_cna"
 ```
 
+#### GithubAdvisories
+```
+  get "/github_advisories", to: "github_advisories#index"
+  get "/github_advisories/:ghsa_id", to: "github_advisories#show"
+```
+
 #### GithubPocs
 ```
   get "/github_pocs", to: "github_pocs#index"

app/controllers/github_advisories_controller.rb

@@ -0,0 +1,10 @@
+class GithubAdvisoriesController < ApplicationController
+  def index
+    @advisories = GithubAdvisory.all
+  end
+
+  def show
+    @advisory = GithubAdivsory.find_by_ghsa_id(params[:ghsa_id])
+    render json: @advisory.to_json
+  end
+end

app/models/github_advisory.rb

@@ -0,0 +1,8 @@
+class GithubAdvisory< ActiveRecord::Base
+    scope :github_reviewed, -> { where("database_specific->>'github_reviewed' = 'true'") }
+    scope :unreviewed, ->  { where("database_specific->>'github_reviewed' = 'false'") }
+
+    def self.find_by_ghsa_id(ghsa_id)
+        find_by(ghsa_id: ghsa_id)
+    end
+end

app/views/github_advisories/index.html.erb

@@ -0,0 +1 @@
+<h1>GithubAdvisories#index</h1>

app/views/github_advisories/show.html.erb

@@ -0,0 +1,2 @@
+<h1> @advisories </h1>
+

config/routes.rb

@@ -30,4 +30,7 @@ Rails.application.routes.draw do
   get "/cnas/cna/:cna_id", to: "cnas#show_for_cna"
   get "/cnas/organization_name/:organization_name", to: "cnas#show_for_orgname"
 
+  get "/github_advisories", to: "github_advisories#index"
+  get "/github_advisories/:ghsa_id", to: "github_advisories#show"
+
 end

db/migrate/20220411174826_create_github_users.rb

@@ -0,0 +1,13 @@
+class CreateGithubUsers < ActiveRecord::Migration[7.0]
+  def change
+    create_table :github_users do |t|
+      t.string :github_id
+      t.string :login
+      t.string :name
+      t.string :avatar_url
+      t.string :bio
+      t.text :bio_html
+      t.string :location
+    end
+  end
+end

db/migrate/20220411181501_create_github_advisories.rb

@@ -0,0 +1,18 @@
+class CreateGithubAdvisories < ActiveRecord::Migration[7.0]
+  def change
+    create_table :github_advisories do |t|
+      t.string :schema_version
+      t.string :ghsa_id
+      t.index :ghsa_id, unique: true
+      t.date :modified
+      t.date :published
+      t.string :aliases, array: true
+      t.string :summary
+      t.string :details
+      t.jsonb :severity
+      t.jsonb :affected
+      t.jsonb :references
+      t.jsonb :database_specific
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -59,6 +59,21 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
     t.index ["cve_id"], name: "index_cves_on_cve_id", unique: true
   end
 
+  create_table "github_advisories", force: :cascade do |t|
+    t.string "schema_version"
+    t.string "ghsa_id"
+    t.date "modified"
+    t.date "published"
+    t.string "aliases", array: true
+    t.string "summary"
+    t.string "details"
+    t.jsonb "severity"
+    t.jsonb "affected"
+    t.jsonb "references"
+    t.jsonb "database_specific"
+    t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true
+  end
+
   create_table "github_pocs", force: :cascade do |t|
     t.integer "github_poc_id"
     t.string "cve_id", default: "None"
@@ -84,6 +99,16 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
     t.index ["github_poc_id"], name: "index_github_pocs_on_github_poc_id", unique: true
   end
 
+  create_table "github_users", force: :cascade do |t|
+    t.string "github_id"
+    t.string "login"
+    t.string "name"
+    t.string "avatar_url"
+    t.string "bio"
+    t.text "bio_html"
+    t.string "location"
+  end
+
   create_table "inthewild_cve_exploits", force: :cascade do |t|
     t.string "cve_id"
     t.string "earliest_report"

db/seeds.rb

@@ -13,6 +13,7 @@ require '/data_importer/lib/importers/inthewild_cve_exploit_importer.rb'
 require '/data_importer/lib/importers/trickest_poc_cve_importer.rb'
 require '/data_importer/lib/importers/cvemon_cve_importer.rb'
 require '/data_importer/lib/importers/cna_importer.rb'
+require '/data_importer/lib/importers/github_advisory_importer.rb'
 
 def line_sep
   puts '----------' * 12
@@ -26,6 +27,7 @@ def perform
     import_cvemon_cves
     import_cpes
     import_cnas
+    import_github_advisories
 end
 
 def import_cves
@@ -43,6 +45,11 @@ def import_github_pocs
     PocInGithubImporter.new.import
 end
 
+def import_github_advisories
+    line_sep
+    GithubAdvisoryImporter.new.import
+end
+
 def import_inthewild_cve_exploits
     line_sep
     InthewildCveExploitImporter.new.import

lib/github_api/security_advisory.rb

@@ -1,5 +1,5 @@
 require '/data_importer/lib/github_api/github_api.rb'
-
+module GithubApi
 class SecurityAdvisory
     SecurityAdvisoryQuery = GithubApi::Client.parse <<-'GRAPHQL'
     query($ghsa_id: String!) {
@@ -53,5 +53,6 @@ class SecurityAdvisory
         end
     end
 end
+end
 
 class QueryExecutionError < StandardError; end

lib/github_api/user.rb

@@ -0,0 +1,29 @@
+require '/data_importer/lib/github_api/github_api.rb'
+module GithubApi
+class User
+    UserProfileQuery = GithubApi::Client.parse <<-'GRAPHQL'
+    query($username: String!) {
+        user(login: $username) {
+          id
+          login
+          name
+          avatarUrl
+          bio
+          bioHTML
+          location
+        }
+      }
+    GRAPHQL
+
+    def self.find(username)
+        response = GithubApi::Client.query(UserProfileQuery, variables: { username: username })
+        if response.errors.any?
+          raise QueryExecutionError.new(response.errors[:data].join(", "))
+        else
+          response.data.user
+        end
+    end
+end
+end
+
+class QueryExecutionError < StandardError; end

lib/importers/cve_list_importer.rb

@@ -2,27 +2,13 @@ require 'git'
 require 'json'
 require 'date'
 require 'bulk_insert'
+require '/data_importer/lib/importers/github_repo.rb'
 
 # This class can be used to import cvelist json data from mitre from their github repo
-class CveListImporter
+class CveListImporter < GithubRepo
-  attr_accessor :repo_url, :repo_path
 
   def initialize
-    @repo_url = 'https://github.com/CVEProject/cvelist.git'
+    super(repo_url='https://github.com/CVEProject/cvelist.git', repo_path='/data_importer/data/cve_list')
-    @repo_path = '/data_importer/data/cve_list'
-  end
-
-  def git_clone_repo
-    Git.clone(repo_url, repo_path)
-  end
-
-  def pull_latest_changes
-    `cd #{repo_path}; git pull;`
-    puts "Now pulling latest changes from #{repo_path}"
-  end
-
-  def read_json(filename)
-    JSON.parse(File.read(filename))
   end
 
   def list_jsons_for_year(year)
@@ -66,12 +52,7 @@ class CveListImporter
   end
 
   def import
-    if Dir.exist?(repo_path)
+    pull_or_clone
-      pull_latest_changes
-    else
-      git_clone_repo
-    end
-
     puts "Now starting import for #{repo_url}."
     puts '----------' * 12
     (1999..Date.today.year).map do |year|

lib/importers/github_advisory_importer.rb

@@ -0,0 +1,77 @@
+require '/data_importer/lib/importers/github_repo.rb'
+
+class GithubAdvisoryImporter < GithubRepo
+    # repo has years that begin with 2017 as first GHSA
+    YEAR_RANGE = (2017..Date.today.year)
+
+    def initialize
+      super(repo_url='https://github.com/github/advisory-database.git', repo_path='/data_importer/data/github_advisories')
+    end
+
+    def advisory_paths
+        advisory_path = "#{repo_path}/advisories"
+        {
+            :base_path => advisory_path,
+            :github_reviewed_path => "#{advisory_path}/github-reviewed",
+            :unreviewed_path => "#{advisory_path}/unreviewed"
+        }
+    end
+
+    def list_jsons_for_year(year)
+        json_wildcard = "*.json"
+        github_reviewed_year_fp = "#{advisory_paths[:github_reviewed_path]}/#{year}/*/*"
+        unreviewed_year_fp = "#{advisory_paths[:unreviewed_path]}/#{year}/*/*"
+        
+        github_reviewed_jsons_fp = Dir["#{github_reviewed_year_fp}/#{json_wildcard}"]
+        unreviewed_jsons_fp = Dir["#{unreviewed_year_fp}/#{json_wildcard}"]
+
+        {
+            :github_reviewed_jsons => github_reviewed_jsons_fp,
+            :unreviewed_jsons => unreviewed_jsons_fp
+        }
+      end
+    
+      def read_jsons_for_year(year)
+        fp_hash = list_jsons_for_year(year)
+        fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons]
+        jsons = fns.map do |fn|
+            read_json(fn)
+        end
+        jsons.flatten
+      end
+
+      def attrs_from_item(json)
+        attrs = {}
+        attrs[:schema_version] = json['schema_version']
+        attrs[:ghsa_id] = json['id']
+        attrs[:modified] = json['modified']
+        attrs[:published] = json['published']
+        attrs[:aliases] = json['aliases']
+        attrs[:summary] = json['summary']
+        attrs[:details] = json['details']
+        attrs[:severity] = json['severity']
+        attrs[:affected] = json['affected']
+        attrs[:references] = json['references']
+        attrs[:database_specific] = json['database_specific']
+        attrs
+    end
+
+    def bulk_insert(jsons)
+        GithubAdvisory.bulk_insert do |worker|
+          jsons.each do |json|
+            attrs = attrs_from_item(json)
+            worker.add(attrs)
+          end
+        end
+    end
+
+    def import
+        pull_or_clone
+        puts "Now importing GithubAdvisories."
+        YEAR_RANGE.each do |year|
+            puts "Importing advisory data from #{year}"
+            jsons = read_jsons_for_year(year)
+            bulk_insert(jsons)
+        end
+    end
+end

lib/importers/github_repo.rb

@@ -0,0 +1,33 @@
+class GithubRepo
+    attr_accessor :repo_url, :repo_path
+
+    def initialize(repo_url=nil, repo_path=nil)
+      @repo_url = repo_url
+      @repo_path = repo_path
+    end
+  
+    def git_clone_repo
+        if repo_url.nil? || repo_path.nil?
+            puts "Please provide a repo url and repo_path"
+        else
+            Git.clone(repo_url, repo_path)
+        end
+    end
+  
+    def pull_latest_changes
+      `cd #{repo_path}; git pull;`
+      puts "Now pulling latest changes from #{repo_path}"
+    end
+  
+    def read_json(filename)
+      JSON.parse(File.read(filename))
+    end
+
+    def pull_or_clone
+        if Dir.exist?(repo_path)
+            pull_latest_changes
+        else
+            git_clone_repo
+        end
+    end
+end