diff --git a/Gemfile b/Gemfile index b721cbd..c7efff7 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,8 @@ gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support gem 'nokogiri' gem 'graphql' gem 'graphql-client' +gem 'rubocop' +gem 'rubocop-rails' # Use postgres as the database for Active Record gem 'pg' diff --git a/Gemfile.lock b/Gemfile.lock index f8ce99d..3ca0aac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,7 @@ GEM public_suffix (>= 2.0.2, < 5.0) archive-zip (0.12.0) io-like (~> 0.3.0) + ast (2.4.2) awesome_print (1.9.2) bindex (0.8.1) bootsnap (1.11.1) @@ -215,6 +216,9 @@ GEM nokogiri (1.13.3) mini_portile2 (~> 2.8.0) racc (~> 1.4) + parallel (1.22.1) + parser (3.1.1.0) + ast (~> 2.4.1) pg (1.3.5) pry (0.14.1) coderay (~> 1.1) @@ -264,6 +268,7 @@ GEM rake (>= 12.2) thor (~> 1.0) zeitwerk (~> 2.5) + rainbow (3.1.1) rake (13.0.6) rb-fsevent (0.11.1) rb-inotify (0.10.1) @@ -276,6 +281,22 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) rexml (3.2.5) + rubocop (1.27.0) + parallel (~> 1.10) + parser (>= 3.1.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml + rubocop-ast (>= 1.16.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.17.0) + parser (>= 3.1.1.0) + rubocop-rails (2.14.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.7.0, < 2.0) + ruby-progressbar (1.11.0) ruby2_keywords (0.0.5) ruby_dep (1.5.0) rubyzip (2.3.2) @@ -331,6 +352,7 @@ GEM unf (0.1.4) unf_ext unf_ext (0.0.8.1) + unicode-display_width (2.1.0) web-console (4.2.0) actionview (>= 6.0.0) activemodel (>= 6.0.0) @@ -375,6 +397,8 @@ DEPENDENCIES rails (~> 7.0.0) railties rest-client + rubocop + rubocop-rails sass-rails selenium-webdriver spring diff --git a/README.md b/README.md index 8222483..e703544 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,9 @@ This is a rails/postgres application that will serve json data from the following data sources: - Cves - Cpes -- CNA Security Advisories -- Github repositories that track public exploits for Cves. +- CNA security advisories +- GHSA Github security advisories +- Github repositories that track public exploits for cves. Check the HTTP API section below for specific endpoints that can be queried via http. @@ -13,6 +14,7 @@ Check the HTTP API section below for specific endpoints that can be queried via - `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format. - `Cna` data from [mitre](https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json). - `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo. +- `GithubAdvisories` data from [github_advisories_database](https://github.com/github/advisory-database/) github repo. - `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed. - `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo. - `CvemonCve` data from [ARPSyndicate](https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json) github repo. @@ -54,6 +56,12 @@ For now unauthenticated api over localhost:3000 until I put in some basic token get "/cnas/cna/:cna_id", to: "cnas#show_for_cna" ``` +#### GithubAdvisories +``` + get "/github_advisories", to: "github_advisories#index" + get "/github_advisories/:ghsa_id", to: "github_advisories#show" +``` + #### GithubPocs ``` get "/github_pocs", to: "github_pocs#index" diff --git a/app/controllers/github_advisories_controller.rb b/app/controllers/github_advisories_controller.rb new file mode 100644 index 0000000..47b4941 --- /dev/null +++ b/app/controllers/github_advisories_controller.rb @@ -0,0 +1,10 @@ +class GithubAdvisoriesController < ApplicationController + def index + @advisories = GithubAdvisory.all + end + + def show + @advisory = GithubAdivsory.find_by_ghsa_id(params[:ghsa_id]) + render json: @advisory.to_json + end +end diff --git a/app/models/github_advisory.rb b/app/models/github_advisory.rb new file mode 100644 index 0000000..6f5e14b --- /dev/null +++ b/app/models/github_advisory.rb @@ -0,0 +1,8 @@ +class GithubAdvisory< ActiveRecord::Base + scope :github_reviewed, -> { where("database_specific->>'github_reviewed' = 'true'") } + scope :unreviewed, -> { where("database_specific->>'github_reviewed' = 'false'") } + + def self.find_by_ghsa_id(ghsa_id) + find_by(ghsa_id: ghsa_id) + end +end diff --git a/app/views/github_advisories/index.html.erb b/app/views/github_advisories/index.html.erb new file mode 100644 index 0000000..c950aa1 --- /dev/null +++ b/app/views/github_advisories/index.html.erb @@ -0,0 +1 @@ +

GithubAdvisories#index

diff --git a/app/views/github_advisories/show.html.erb b/app/views/github_advisories/show.html.erb new file mode 100644 index 0000000..445887a --- /dev/null +++ b/app/views/github_advisories/show.html.erb @@ -0,0 +1,2 @@ +

@advisories

+ diff --git a/config/routes.rb b/config/routes.rb index 7da965a..7a436ff 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -30,4 +30,7 @@ Rails.application.routes.draw do get "/cnas/cna/:cna_id", to: "cnas#show_for_cna" get "/cnas/organization_name/:organization_name", to: "cnas#show_for_orgname" + get "/github_advisories", to: "github_advisories#index" + get "/github_advisories/:ghsa_id", to: "github_advisories#show" + end diff --git a/db/migrate/20220411174826_create_github_users.rb b/db/migrate/20220411174826_create_github_users.rb new file mode 100644 index 0000000..42d8368 --- /dev/null +++ b/db/migrate/20220411174826_create_github_users.rb @@ -0,0 +1,13 @@ +class CreateGithubUsers < ActiveRecord::Migration[7.0] + def change + create_table :github_users do |t| + t.string :github_id + t.string :login + t.string :name + t.string :avatar_url + t.string :bio + t.text :bio_html + t.string :location + end + end +end diff --git a/db/migrate/20220411181501_create_github_advisories.rb b/db/migrate/20220411181501_create_github_advisories.rb new file mode 100644 index 0000000..a3eac95 --- /dev/null +++ b/db/migrate/20220411181501_create_github_advisories.rb @@ -0,0 +1,18 @@ +class CreateGithubAdvisories < ActiveRecord::Migration[7.0] + def change + create_table :github_advisories do |t| + t.string :schema_version + t.string :ghsa_id + t.index :ghsa_id, unique: true + t.date :modified + t.date :published + t.string :aliases, array: true + t.string :summary + t.string :details + t.jsonb :severity + t.jsonb :affected + t.jsonb :references + t.jsonb :database_specific + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 90439dd..f808210 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do +ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -59,6 +59,21 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do t.index ["cve_id"], name: "index_cves_on_cve_id", unique: true end + create_table "github_advisories", force: :cascade do |t| + t.string "schema_version" + t.string "ghsa_id" + t.date "modified" + t.date "published" + t.string "aliases", array: true + t.string "summary" + t.string "details" + t.jsonb "severity" + t.jsonb "affected" + t.jsonb "references" + t.jsonb "database_specific" + t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true + end + create_table "github_pocs", force: :cascade do |t| t.integer "github_poc_id" t.string "cve_id", default: "None" @@ -84,6 +99,16 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_223152) do t.index ["github_poc_id"], name: "index_github_pocs_on_github_poc_id", unique: true end + create_table "github_users", force: :cascade do |t| + t.string "github_id" + t.string "login" + t.string "name" + t.string "avatar_url" + t.string "bio" + t.text "bio_html" + t.string "location" + end + create_table "inthewild_cve_exploits", force: :cascade do |t| t.string "cve_id" t.string "earliest_report" diff --git a/db/seeds.rb b/db/seeds.rb index c51e355..0354ecd 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -13,6 +13,7 @@ require '/data_importer/lib/importers/inthewild_cve_exploit_importer.rb' require '/data_importer/lib/importers/trickest_poc_cve_importer.rb' require '/data_importer/lib/importers/cvemon_cve_importer.rb' require '/data_importer/lib/importers/cna_importer.rb' +require '/data_importer/lib/importers/github_advisory_importer.rb' def line_sep puts '----------' * 12 @@ -26,6 +27,7 @@ def perform import_cvemon_cves import_cpes import_cnas + import_github_advisories end def import_cves @@ -43,6 +45,11 @@ def import_github_pocs PocInGithubImporter.new.import end +def import_github_advisories + line_sep + GithubAdvisoryImporter.new.import +end + def import_inthewild_cve_exploits line_sep InthewildCveExploitImporter.new.import diff --git a/lib/github_api/security_advisory.rb b/lib/github_api/security_advisory.rb index 6f60f18..011dde0 100644 --- a/lib/github_api/security_advisory.rb +++ b/lib/github_api/security_advisory.rb @@ -1,5 +1,5 @@ require '/data_importer/lib/github_api/github_api.rb' - +module GithubApi class SecurityAdvisory SecurityAdvisoryQuery = GithubApi::Client.parse <<-'GRAPHQL' query($ghsa_id: String!) { @@ -53,5 +53,6 @@ class SecurityAdvisory end end end +end class QueryExecutionError < StandardError; end \ No newline at end of file diff --git a/lib/github_api/user.rb b/lib/github_api/user.rb new file mode 100644 index 0000000..cdbdc22 --- /dev/null +++ b/lib/github_api/user.rb @@ -0,0 +1,29 @@ +require '/data_importer/lib/github_api/github_api.rb' +module GithubApi +class User + UserProfileQuery = GithubApi::Client.parse <<-'GRAPHQL' + query($username: String!) { + user(login: $username) { + id + login + name + avatarUrl + bio + bioHTML + location + } + } + GRAPHQL + + def self.find(username) + response = GithubApi::Client.query(UserProfileQuery, variables: { username: username }) + if response.errors.any? + raise QueryExecutionError.new(response.errors[:data].join(", ")) + else + response.data.user + end + end +end +end + +class QueryExecutionError < StandardError; end \ No newline at end of file diff --git a/lib/importers/cve_list_importer.rb b/lib/importers/cve_list_importer.rb index 7889769..27f0cf7 100644 --- a/lib/importers/cve_list_importer.rb +++ b/lib/importers/cve_list_importer.rb @@ -2,27 +2,13 @@ require 'git' require 'json' require 'date' require 'bulk_insert' +require '/data_importer/lib/importers/github_repo.rb' # This class can be used to import cvelist json data from mitre from their github repo -class CveListImporter - attr_accessor :repo_url, :repo_path +class CveListImporter < GithubRepo def initialize - @repo_url = 'https://github.com/CVEProject/cvelist.git' - @repo_path = '/data_importer/data/cve_list' - end - - def git_clone_repo - Git.clone(repo_url, repo_path) - end - - def pull_latest_changes - `cd #{repo_path}; git pull;` - puts "Now pulling latest changes from #{repo_path}" - end - - def read_json(filename) - JSON.parse(File.read(filename)) + super(repo_url='https://github.com/CVEProject/cvelist.git', repo_path='/data_importer/data/cve_list') end def list_jsons_for_year(year) @@ -66,12 +52,7 @@ class CveListImporter end def import - if Dir.exist?(repo_path) - pull_latest_changes - else - git_clone_repo - end - + pull_or_clone puts "Now starting import for #{repo_url}." puts '----------' * 12 (1999..Date.today.year).map do |year| diff --git a/lib/importers/github_advisory_importer.rb b/lib/importers/github_advisory_importer.rb new file mode 100644 index 0000000..f6e7df3 --- /dev/null +++ b/lib/importers/github_advisory_importer.rb @@ -0,0 +1,77 @@ +require '/data_importer/lib/importers/github_repo.rb' + +class GithubAdvisoryImporter < GithubRepo + # repo has years that begin with 2017 as first GHSA + YEAR_RANGE = (2017..Date.today.year) + + def initialize + super(repo_url='https://github.com/github/advisory-database.git', repo_path='/data_importer/data/github_advisories') + end + + def advisory_paths + advisory_path = "#{repo_path}/advisories" + { + :base_path => advisory_path, + :github_reviewed_path => "#{advisory_path}/github-reviewed", + :unreviewed_path => "#{advisory_path}/unreviewed" + } + end + + def list_jsons_for_year(year) + json_wildcard = "*.json" + github_reviewed_year_fp = "#{advisory_paths[:github_reviewed_path]}/#{year}/*/*" + unreviewed_year_fp = "#{advisory_paths[:unreviewed_path]}/#{year}/*/*" + + github_reviewed_jsons_fp = Dir["#{github_reviewed_year_fp}/#{json_wildcard}"] + unreviewed_jsons_fp = Dir["#{unreviewed_year_fp}/#{json_wildcard}"] + + { + :github_reviewed_jsons => github_reviewed_jsons_fp, + :unreviewed_jsons => unreviewed_jsons_fp + } + end + + def read_jsons_for_year(year) + fp_hash = list_jsons_for_year(year) + fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons] + jsons = fns.map do |fn| + read_json(fn) + end + jsons.flatten + end + + def attrs_from_item(json) + attrs = {} + attrs[:schema_version] = json['schema_version'] + attrs[:ghsa_id] = json['id'] + attrs[:modified] = json['modified'] + attrs[:published] = json['published'] + attrs[:aliases] = json['aliases'] + attrs[:summary] = json['summary'] + attrs[:details] = json['details'] + attrs[:severity] = json['severity'] + attrs[:affected] = json['affected'] + attrs[:references] = json['references'] + attrs[:database_specific] = json['database_specific'] + attrs + end + + def bulk_insert(jsons) + GithubAdvisory.bulk_insert do |worker| + jsons.each do |json| + attrs = attrs_from_item(json) + worker.add(attrs) + end + end + end + + def import + pull_or_clone + puts "Now importing GithubAdvisories." + YEAR_RANGE.each do |year| + puts "Importing advisory data from #{year}" + jsons = read_jsons_for_year(year) + bulk_insert(jsons) + end + end +end \ No newline at end of file diff --git a/lib/importers/github_repo.rb b/lib/importers/github_repo.rb new file mode 100644 index 0000000..0f30b42 --- /dev/null +++ b/lib/importers/github_repo.rb @@ -0,0 +1,33 @@ +class GithubRepo + attr_accessor :repo_url, :repo_path + + def initialize(repo_url=nil, repo_path=nil) + @repo_url = repo_url + @repo_path = repo_path + end + + def git_clone_repo + if repo_url.nil? || repo_path.nil? + puts "Please provide a repo url and repo_path" + else + Git.clone(repo_url, repo_path) + end + end + + def pull_latest_changes + `cd #{repo_path}; git pull;` + puts "Now pulling latest changes from #{repo_path}" + end + + def read_json(filename) + JSON.parse(File.read(filename)) + end + + def pull_or_clone + if Dir.exist?(repo_path) + pull_latest_changes + else + git_clone_repo + end + end +end \ No newline at end of file diff --git a/vendor/.keep b/vendor/.keep deleted file mode 100644 index e69de29..0000000