diff --git a/Gemfile b/Gemfile
index b721cbd..c7efff7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -14,6 +14,8 @@ gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support
gem 'nokogiri'
gem 'graphql'
gem 'graphql-client'
+gem 'rubocop'
+gem 'rubocop-rails'
# Use postgres as the database for Active Record
gem 'pg'
diff --git a/Gemfile.lock b/Gemfile.lock
index f8ce99d..3ca0aac 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -79,6 +79,7 @@ GEM
public_suffix (>= 2.0.2, < 5.0)
archive-zip (0.12.0)
io-like (~> 0.3.0)
+ ast (2.4.2)
awesome_print (1.9.2)
bindex (0.8.1)
bootsnap (1.11.1)
@@ -215,6 +216,9 @@ GEM
nokogiri (1.13.3)
mini_portile2 (~> 2.8.0)
racc (~> 1.4)
+ parallel (1.22.1)
+ parser (3.1.1.0)
+ ast (~> 2.4.1)
pg (1.3.5)
pry (0.14.1)
coderay (~> 1.1)
@@ -264,6 +268,7 @@ GEM
rake (>= 12.2)
thor (~> 1.0)
zeitwerk (~> 2.5)
+ rainbow (3.1.1)
rake (13.0.6)
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
@@ -276,6 +281,22 @@ GEM
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
rexml (3.2.5)
+ rubocop (1.27.0)
+ parallel (~> 1.10)
+ parser (>= 3.1.0.0)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml
+ rubocop-ast (>= 1.16.0, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 1.4.0, < 3.0)
+ rubocop-ast (1.17.0)
+ parser (>= 3.1.1.0)
+ rubocop-rails (2.14.2)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.7.0, < 2.0)
+ ruby-progressbar (1.11.0)
ruby2_keywords (0.0.5)
ruby_dep (1.5.0)
rubyzip (2.3.2)
@@ -331,6 +352,7 @@ GEM
unf (0.1.4)
unf_ext
unf_ext (0.0.8.1)
+ unicode-display_width (2.1.0)
web-console (4.2.0)
actionview (>= 6.0.0)
activemodel (>= 6.0.0)
@@ -375,6 +397,8 @@ DEPENDENCIES
rails (~> 7.0.0)
railties
rest-client
+ rubocop
+ rubocop-rails
sass-rails
selenium-webdriver
spring
diff --git a/README.md b/README.md
index 8222483..e703544 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,9 @@
This is a rails/postgres application that will serve json data from the following data sources:
- Cves
- Cpes
-- CNA Security Advisories
-- Github repositories that track public exploits for Cves.
+- CNA security advisories
+- GHSA Github security advisories
+- Github repositories that track public exploits for cves.
Check the HTTP API section below for specific endpoints that can be queried via http.
@@ -13,6 +14,7 @@ Check the HTTP API section below for specific endpoints that can be queried via
- `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format.
- `Cna` data from [mitre](https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json).
- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo.
+- `GithubAdvisories` data from [github_advisories_database](https://github.com/github/advisory-database/) github repo.
- `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed.
- `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo.
- `CvemonCve` data from [ARPSyndicate](https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json) github repo.
@@ -54,6 +56,12 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
get "/cnas/cna/:cna_id", to: "cnas#show_for_cna"
```
+#### GithubAdvisories
+```
+ get "/github_advisories", to: "github_advisories#index"
+ get "/github_advisories/:ghsa_id", to: "github_advisories#show"
+```
+
#### GithubPocs
```
get "/github_pocs", to: "github_pocs#index"
diff --git a/app/controllers/github_advisories_controller.rb b/app/controllers/github_advisories_controller.rb
new file mode 100644
index 0000000..47b4941
--- /dev/null
+++ b/app/controllers/github_advisories_controller.rb
@@ -0,0 +1,10 @@
+class GithubAdvisoriesController < ApplicationController
+ def index
+ @advisories = GithubAdvisory.all
+ end
+
+ def show
+ @advisory = GithubAdivsory.find_by_ghsa_id(params[:ghsa_id])
+ render json: @advisory.to_json
+ end
+end
diff --git a/app/models/github_advisory.rb b/app/models/github_advisory.rb
new file mode 100644
index 0000000..6f5e14b
--- /dev/null
+++ b/app/models/github_advisory.rb
@@ -0,0 +1,8 @@
+class GithubAdvisory< ActiveRecord::Base
+ scope :github_reviewed, -> { where("database_specific->>'github_reviewed' = 'true'") }
+ scope :unreviewed, -> { where("database_specific->>'github_reviewed' = 'false'") }
+
+ def self.find_by_ghsa_id(ghsa_id)
+ find_by(ghsa_id: ghsa_id)
+ end
+end
diff --git a/app/views/github_advisories/index.html.erb b/app/views/github_advisories/index.html.erb
new file mode 100644
index 0000000..c950aa1
--- /dev/null
+++ b/app/views/github_advisories/index.html.erb
@@ -0,0 +1 @@
+
GithubAdvisories#index
diff --git a/app/views/github_advisories/show.html.erb b/app/views/github_advisories/show.html.erb
new file mode 100644
index 0000000..445887a
--- /dev/null
+++ b/app/views/github_advisories/show.html.erb
@@ -0,0 +1,2 @@
+ @advisories
+
diff --git a/config/routes.rb b/config/routes.rb
index 7da965a..7a436ff 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -30,4 +30,7 @@ Rails.application.routes.draw do
get "/cnas/cna/:cna_id", to: "cnas#show_for_cna"
get "/cnas/organization_name/:organization_name", to: "cnas#show_for_orgname"
+ get "/github_advisories", to: "github_advisories#index"
+ get "/github_advisories/:ghsa_id", to: "github_advisories#show"
+
end
diff --git a/db/migrate/20220411174826_create_github_users.rb b/db/migrate/20220411174826_create_github_users.rb
new file mode 100644
index 0000000..42d8368
--- /dev/null
+++ b/db/migrate/20220411174826_create_github_users.rb
@@ -0,0 +1,13 @@
+class CreateGithubUsers < ActiveRecord::Migration[7.0]
+ def change
+ create_table :github_users do |t|
+ t.string :github_id
+ t.string :login
+ t.string :name
+ t.string :avatar_url
+ t.string :bio
+ t.text :bio_html
+ t.string :location
+ end
+ end
+end
diff --git a/db/migrate/20220411181501_create_github_advisories.rb b/db/migrate/20220411181501_create_github_advisories.rb
new file mode 100644
index 0000000..a3eac95
--- /dev/null
+++ b/db/migrate/20220411181501_create_github_advisories.rb
@@ -0,0 +1,18 @@
+class CreateGithubAdvisories < ActiveRecord::Migration[7.0]
+ def change
+ create_table :github_advisories do |t|
+ t.string :schema_version
+ t.string :ghsa_id
+ t.index :ghsa_id, unique: true
+ t.date :modified
+ t.date :published
+ t.string :aliases, array: true
+ t.string :summary
+ t.string :details
+ t.jsonb :severity
+ t.jsonb :affected
+ t.jsonb :references
+ t.jsonb :database_specific
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 90439dd..f808210 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -59,6 +59,21 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
t.index ["cve_id"], name: "index_cves_on_cve_id", unique: true
end
+ create_table "github_advisories", force: :cascade do |t|
+ t.string "schema_version"
+ t.string "ghsa_id"
+ t.date "modified"
+ t.date "published"
+ t.string "aliases", array: true
+ t.string "summary"
+ t.string "details"
+ t.jsonb "severity"
+ t.jsonb "affected"
+ t.jsonb "references"
+ t.jsonb "database_specific"
+ t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true
+ end
+
create_table "github_pocs", force: :cascade do |t|
t.integer "github_poc_id"
t.string "cve_id", default: "None"
@@ -84,6 +99,16 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do
t.index ["github_poc_id"], name: "index_github_pocs_on_github_poc_id", unique: true
end
+ create_table "github_users", force: :cascade do |t|
+ t.string "github_id"
+ t.string "login"
+ t.string "name"
+ t.string "avatar_url"
+ t.string "bio"
+ t.text "bio_html"
+ t.string "location"
+ end
+
create_table "inthewild_cve_exploits", force: :cascade do |t|
t.string "cve_id"
t.string "earliest_report"
diff --git a/db/seeds.rb b/db/seeds.rb
index c51e355..0354ecd 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -13,6 +13,7 @@ require '/data_importer/lib/importers/inthewild_cve_exploit_importer.rb'
require '/data_importer/lib/importers/trickest_poc_cve_importer.rb'
require '/data_importer/lib/importers/cvemon_cve_importer.rb'
require '/data_importer/lib/importers/cna_importer.rb'
+require '/data_importer/lib/importers/github_advisory_importer.rb'
def line_sep
puts '----------' * 12
@@ -26,6 +27,7 @@ def perform
import_cvemon_cves
import_cpes
import_cnas
+ import_github_advisories
end
def import_cves
@@ -43,6 +45,11 @@ def import_github_pocs
PocInGithubImporter.new.import
end
+def import_github_advisories
+ line_sep
+ GithubAdvisoryImporter.new.import
+end
+
def import_inthewild_cve_exploits
line_sep
InthewildCveExploitImporter.new.import
diff --git a/lib/github_api/security_advisory.rb b/lib/github_api/security_advisory.rb
index 6f60f18..011dde0 100644
--- a/lib/github_api/security_advisory.rb
+++ b/lib/github_api/security_advisory.rb
@@ -1,5 +1,5 @@
require '/data_importer/lib/github_api/github_api.rb'
-
+module GithubApi
class SecurityAdvisory
SecurityAdvisoryQuery = GithubApi::Client.parse <<-'GRAPHQL'
query($ghsa_id: String!) {
@@ -53,5 +53,6 @@ class SecurityAdvisory
end
end
end
+end
class QueryExecutionError < StandardError; end
\ No newline at end of file
diff --git a/lib/github_api/user.rb b/lib/github_api/user.rb
new file mode 100644
index 0000000..cdbdc22
--- /dev/null
+++ b/lib/github_api/user.rb
@@ -0,0 +1,29 @@
+require '/data_importer/lib/github_api/github_api.rb'
+module GithubApi
+class User
+ UserProfileQuery = GithubApi::Client.parse <<-'GRAPHQL'
+ query($username: String!) {
+ user(login: $username) {
+ id
+ login
+ name
+ avatarUrl
+ bio
+ bioHTML
+ location
+ }
+ }
+ GRAPHQL
+
+ def self.find(username)
+ response = GithubApi::Client.query(UserProfileQuery, variables: { username: username })
+ if response.errors.any?
+ raise QueryExecutionError.new(response.errors[:data].join(", "))
+ else
+ response.data.user
+ end
+ end
+end
+end
+
+class QueryExecutionError < StandardError; end
\ No newline at end of file
diff --git a/lib/importers/cve_list_importer.rb b/lib/importers/cve_list_importer.rb
index 7889769..27f0cf7 100644
--- a/lib/importers/cve_list_importer.rb
+++ b/lib/importers/cve_list_importer.rb
@@ -2,27 +2,13 @@ require 'git'
require 'json'
require 'date'
require 'bulk_insert'
+require '/data_importer/lib/importers/github_repo.rb'
# This class can be used to import cvelist json data from mitre from their github repo
-class CveListImporter
- attr_accessor :repo_url, :repo_path
+class CveListImporter < GithubRepo
def initialize
- @repo_url = 'https://github.com/CVEProject/cvelist.git'
- @repo_path = '/data_importer/data/cve_list'
- end
-
- def git_clone_repo
- Git.clone(repo_url, repo_path)
- end
-
- def pull_latest_changes
- `cd #{repo_path}; git pull;`
- puts "Now pulling latest changes from #{repo_path}"
- end
-
- def read_json(filename)
- JSON.parse(File.read(filename))
+ super(repo_url='https://github.com/CVEProject/cvelist.git', repo_path='/data_importer/data/cve_list')
end
def list_jsons_for_year(year)
@@ -66,12 +52,7 @@ class CveListImporter
end
def import
- if Dir.exist?(repo_path)
- pull_latest_changes
- else
- git_clone_repo
- end
-
+ pull_or_clone
puts "Now starting import for #{repo_url}."
puts '----------' * 12
(1999..Date.today.year).map do |year|
diff --git a/lib/importers/github_advisory_importer.rb b/lib/importers/github_advisory_importer.rb
new file mode 100644
index 0000000..f6e7df3
--- /dev/null
+++ b/lib/importers/github_advisory_importer.rb
@@ -0,0 +1,77 @@
+require '/data_importer/lib/importers/github_repo.rb'
+
+class GithubAdvisoryImporter < GithubRepo
+ # repo has years that begin with 2017 as first GHSA
+ YEAR_RANGE = (2017..Date.today.year)
+
+ def initialize
+ super(repo_url='https://github.com/github/advisory-database.git', repo_path='/data_importer/data/github_advisories')
+ end
+
+ def advisory_paths
+ advisory_path = "#{repo_path}/advisories"
+ {
+ :base_path => advisory_path,
+ :github_reviewed_path => "#{advisory_path}/github-reviewed",
+ :unreviewed_path => "#{advisory_path}/unreviewed"
+ }
+ end
+
+ def list_jsons_for_year(year)
+ json_wildcard = "*.json"
+ github_reviewed_year_fp = "#{advisory_paths[:github_reviewed_path]}/#{year}/*/*"
+ unreviewed_year_fp = "#{advisory_paths[:unreviewed_path]}/#{year}/*/*"
+
+ github_reviewed_jsons_fp = Dir["#{github_reviewed_year_fp}/#{json_wildcard}"]
+ unreviewed_jsons_fp = Dir["#{unreviewed_year_fp}/#{json_wildcard}"]
+
+ {
+ :github_reviewed_jsons => github_reviewed_jsons_fp,
+ :unreviewed_jsons => unreviewed_jsons_fp
+ }
+ end
+
+ def read_jsons_for_year(year)
+ fp_hash = list_jsons_for_year(year)
+ fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons]
+ jsons = fns.map do |fn|
+ read_json(fn)
+ end
+ jsons.flatten
+ end
+
+ def attrs_from_item(json)
+ attrs = {}
+ attrs[:schema_version] = json['schema_version']
+ attrs[:ghsa_id] = json['id']
+ attrs[:modified] = json['modified']
+ attrs[:published] = json['published']
+ attrs[:aliases] = json['aliases']
+ attrs[:summary] = json['summary']
+ attrs[:details] = json['details']
+ attrs[:severity] = json['severity']
+ attrs[:affected] = json['affected']
+ attrs[:references] = json['references']
+ attrs[:database_specific] = json['database_specific']
+ attrs
+ end
+
+ def bulk_insert(jsons)
+ GithubAdvisory.bulk_insert do |worker|
+ jsons.each do |json|
+ attrs = attrs_from_item(json)
+ worker.add(attrs)
+ end
+ end
+ end
+
+ def import
+ pull_or_clone
+ puts "Now importing GithubAdvisories."
+ YEAR_RANGE.each do |year|
+ puts "Importing advisory data from #{year}"
+ jsons = read_jsons_for_year(year)
+ bulk_insert(jsons)
+ end
+ end
+end
\ No newline at end of file
diff --git a/lib/importers/github_repo.rb b/lib/importers/github_repo.rb
new file mode 100644
index 0000000..0f30b42
--- /dev/null
+++ b/lib/importers/github_repo.rb
@@ -0,0 +1,33 @@
+class GithubRepo
+ attr_accessor :repo_url, :repo_path
+
+ def initialize(repo_url=nil, repo_path=nil)
+ @repo_url = repo_url
+ @repo_path = repo_path
+ end
+
+ def git_clone_repo
+ if repo_url.nil? || repo_path.nil?
+ puts "Please provide a repo url and repo_path"
+ else
+ Git.clone(repo_url, repo_path)
+ end
+ end
+
+ def pull_latest_changes
+ `cd #{repo_path}; git pull;`
+ puts "Now pulling latest changes from #{repo_path}"
+ end
+
+ def read_json(filename)
+ JSON.parse(File.read(filename))
+ end
+
+ def pull_or_clone
+ if Dir.exist?(repo_path)
+ pull_latest_changes
+ else
+ git_clone_repo
+ end
+ end
+end
\ No newline at end of file
diff --git a/vendor/.keep b/vendor/.keep
deleted file mode 100644
index e69de29..0000000