From 73cdaabe3aaa5678bf66d7e6fc57d139c3c5182b Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Thu, 7 Apr 2022 01:51:44 -0500 Subject: [PATCH] add support for trickest poc cve github repo --- .gitignore | 3 ++ README.md | 14 ++++++- app/controllers/github_pocs_controller.rb | 5 +++ .../trickest_poc_cves_controller.rb | 21 ++++++++++ app/models/github_poc.rb | 3 ++ app/models/trickest_poc_cve.rb | 5 +++ app/views/trickest_poc_cves/index.html.erb | 1 + app/views/trickest_poc_cves/show.html.erb | 2 + config/routes.rb | 7 ++++ ...20220407051821_create_trickest_poc_cves.rb | 10 +++++ db/schema.rb | 9 +++- db/seeds.rb | 7 ++++ lib/cve_list_importer.rb | 3 +- lib/poc_in_github_importer.rb | 2 +- lib/trickest_poc_cve_importer.rb | 42 ++++++++----------- 15 files changed, 104 insertions(+), 30 deletions(-) create mode 100644 app/controllers/trickest_poc_cves_controller.rb create mode 100644 app/models/trickest_poc_cve.rb create mode 100644 app/views/trickest_poc_cves/index.html.erb create mode 100644 app/views/trickest_poc_cves/show.html.erb create mode 100644 db/migrate/20220407051821_create_trickest_poc_cves.rb diff --git a/.gitignore b/.gitignore index 9e1b5ec..ecca9c5 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ # Any API keys or envars we dont want to commit add here. /twitter_credentials.env + +# Ignore our data dir as that gets populated during initial seed/setup +/data/* diff --git a/README.md b/README.md index 1f3aa23..30152e1 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ Import common security data such as CVE, CPE, and Security Advisories from various CNAs into a rails app with a postgresql db backend. ## Supported data models: -- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) mitre. +- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) github repo. - `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format. -- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub). +- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo. - `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed. +- `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo. ## Initial Setup @@ -43,10 +44,19 @@ For now unauthenticated api over localhost:3000 until I put in some basic token get "/github_pocs", to: "github_pocs#index" get "/github_pocs/:id", to: "github_pocs#show" get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve" + get "/github_pocs/years/:year", to: "github_pocs#show_year" ``` #### InthewildCveExploits ``` get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index" get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show" +``` + +#### TrickestPocCves +``` + get "/trickest_poc_cves", to: "trickest_poc_cves#index" + get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show" + get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve" + get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year" ``` \ No newline at end of file diff --git a/app/controllers/github_pocs_controller.rb b/app/controllers/github_pocs_controller.rb index 79861ca..3f11751 100644 --- a/app/controllers/github_pocs_controller.rb +++ b/app/controllers/github_pocs_controller.rb @@ -13,4 +13,9 @@ class GithubPocsController < ApplicationController render json: @poc.to_json end + def show_year + @cves_for_year = GithubPoc.from_year(params[:year]) + render json: @cves_for_year.to_json + end + end diff --git a/app/controllers/trickest_poc_cves_controller.rb b/app/controllers/trickest_poc_cves_controller.rb new file mode 100644 index 0000000..1e0b876 --- /dev/null +++ b/app/controllers/trickest_poc_cves_controller.rb @@ -0,0 +1,21 @@ +class TrickestPocCvesController < ApplicationController + def index + @pocs = TrickestPocCve.all + end + + def show + @poc = TrickestPocCve.find_by(:id => params[:id]) + render json: @poc.to_json + end + + def show_for_cve + @poc = TrickestPocCve.where(:cve_id => params[:cve_id]) + render json: @poc.to_json + end + + def show_year + @cves_for_year = TrickestPocCve.from_year(params[:year]) + render json: @cves_for_year.to_json + end + +end diff --git a/app/models/github_poc.rb b/app/models/github_poc.rb index 52d763c..f804d89 100644 --- a/app/models/github_poc.rb +++ b/app/models/github_poc.rb @@ -1,2 +1,5 @@ class GithubPoc < ActiveRecord::Base + def self.from_year(year) + where("cve_id LIKE ?", "CVE-#{year}-%") + end end diff --git a/app/models/trickest_poc_cve.rb b/app/models/trickest_poc_cve.rb new file mode 100644 index 0000000..4aa7d69 --- /dev/null +++ b/app/models/trickest_poc_cve.rb @@ -0,0 +1,5 @@ +class TrickestPocCve < ActiveRecord::Base + def self.from_year(year) + where("cve_id LIKE ?", "CVE-#{year}-%") + end +end diff --git a/app/views/trickest_poc_cves/index.html.erb b/app/views/trickest_poc_cves/index.html.erb new file mode 100644 index 0000000..8874e23 --- /dev/null +++ b/app/views/trickest_poc_cves/index.html.erb @@ -0,0 +1 @@ +

TrickestPocCves#index

diff --git a/app/views/trickest_poc_cves/show.html.erb b/app/views/trickest_poc_cves/show.html.erb new file mode 100644 index 0000000..740ff12 --- /dev/null +++ b/app/views/trickest_poc_cves/show.html.erb @@ -0,0 +1,2 @@ +

@poc

+ diff --git a/config/routes.rb b/config/routes.rb index 0ed9775..24b1d34 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -10,7 +10,14 @@ Rails.application.routes.draw do get "/github_pocs", to: "github_pocs#index" get "/github_pocs/:id", to: "github_pocs#show" get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve" + get "/github_pocs/years/:year", to: "github_pocs#show_year" get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index" get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show" + + get "/trickest_poc_cves", to: "trickest_poc_cves#index" + get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show" + get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve" + get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year" + end diff --git a/db/migrate/20220407051821_create_trickest_poc_cves.rb b/db/migrate/20220407051821_create_trickest_poc_cves.rb new file mode 100644 index 0000000..8fda9fd --- /dev/null +++ b/db/migrate/20220407051821_create_trickest_poc_cves.rb @@ -0,0 +1,10 @@ +class CreateTrickestPocCves < ActiveRecord::Migration[7.0] + def change + create_table :trickest_poc_cves do |t| + t.string :cve_id + t.string :cve_url + t.string :description + t.string :poc_links, array: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 972c76e..1c8580f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do +ActiveRecord::Schema[7.0].define(version: 2022_04_07_051821) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -69,4 +69,11 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do t.string "earliest_report" end + create_table "trickest_poc_cves", force: :cascade do |t| + t.string "cve_id" + t.string "cve_url" + t.string "description" + t.string "poc_links", array: true + end + end diff --git a/db/seeds.rb b/db/seeds.rb index 6598da4..db6c918 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -10,6 +10,7 @@ require '/data_importer/lib/cpe_importer.rb' require '/data_importer/lib/cve_list_importer.rb' require '/data_importer/lib/poc_in_github_importer.rb' require '/data_importer/lib/inthewild_cve_exploit_importer.rb' +require '/data_importer/lib/trickest_poc_cve_importer.rb' def line_sep puts '----------' * 12 @@ -18,6 +19,7 @@ end def perform import_cves import_github_pocs + import_trickest_poc_cves import_inthewild_cve_exploits import_cpes end @@ -42,4 +44,9 @@ def import_inthewild_cve_exploits InthewildCveExploitImporter.new.import end +def import_trickest_poc_cves + line_sep + TrickestPocCveImporter.new.import +end + perform \ No newline at end of file diff --git a/lib/cve_list_importer.rb b/lib/cve_list_importer.rb index f9cc64f..7889769 100644 --- a/lib/cve_list_importer.rb +++ b/lib/cve_list_importer.rb @@ -72,7 +72,8 @@ class CveListImporter git_clone_repo end - puts "Now starting import for CveList." + puts "Now starting import for #{repo_url}." + puts '----------' * 12 (1999..Date.today.year).map do |year| cves_from_json = cves_for_year(year) diff --git a/lib/poc_in_github_importer.rb b/lib/poc_in_github_importer.rb index 5887cfe..bf7a1c8 100644 --- a/lib/poc_in_github_importer.rb +++ b/lib/poc_in_github_importer.rb @@ -106,7 +106,7 @@ class PocInGithubImporter git_clone_repo end - puts "Now starting import for PocInGithub." + puts "Now starting import for #{repo_url}." (1999..Date.today.year).map do |year| cves_from_json = cves_for_year(year) diff --git a/lib/trickest_poc_cve_importer.rb b/lib/trickest_poc_cve_importer.rb index 3bcfe38..8f5d58b 100644 --- a/lib/trickest_poc_cve_importer.rb +++ b/lib/trickest_poc_cve_importer.rb @@ -50,15 +50,12 @@ class TrickestPocCveImporter p_text = doc.xpath('//p').map {|p| p.text } links_for_poc = doc.xpath('//p/a').map {|a| a.values}.flatten - data_hash["#{cve_id}"] = cve_url + data_hash['cve_id'] = cve_id + data_hash['cve_url'] = cve_url # p_text[0] is always an ' '. - data_hash['Description'] = p_text[1] + data_hash['description'] = p_text[1] - # array of values if its a links. hard to distinguish between ones under POC and ones under Github - # if it contains no data under the heading there will be no .value but instead .text will return data. - # these ones can both have multiple values - # just normalize and put POC and Github stuff under one key now. idc i just need the URL - data_hash['POC'] = links_for_poc + data_hash['poc_links'] = links_for_poc data_hash end @@ -80,25 +77,19 @@ class TrickestPocCveImporter def cve_attrs_from_item(json) cve_attrs = {} - #cve_attrs[:cve_data_meta] = json['CVE_data_meta'] - #cve_attrs[:cve_id] = json['CVE_data_meta']['ID'] - #cve_attrs[:affects] = json['affects'] - #cve_attrs[:data_format] = json['data_format'] - #cve_attrs[:data_type] = json['data_type'] - #cve_attrs[:data_version] = json['data_version'] - #cve_attrs[:description] = json['description'] - #cve_attrs[:impact] = json['impact'] - #cve_attrs[:problemtype] = json['problemtype'] - #cve_attrs[:references] = json['references'] - #cve_attrs[:source] = json['source'] + cve_attrs[:cve_id] = json['cve_id'] + cve_attrs[:cve_url] = json['cve_url'] + cve_attrs[:description] = json['description'] + cve_attrs[:poc_links] = json['poc_links'] cve_attrs end # for bulk inserting def cves_for_year(year) - json_data = read_jsons_for_year(year) - json_data.map do |json_f| - cve_attrs_from_item(json_f) + htmls = read_mds_for_year(year) + htmls.map do |html| + data_hash = html_to_hash(html) + cve_attrs_from_item(data_hash) end end @@ -109,15 +100,16 @@ class TrickestPocCveImporter git_clone_repo end - puts "Now starting import for CveList." + puts "Now starting import for #{repo_url}." + puts '----------' * 12 (1999..Date.today.year).map do |year| - cves_from_json = cves_for_year(year) + cves_from_markdown = cves_for_year(year) - ids = cves_from_json.map { |cve| cve[:cve_id] } + ids = cves_from_markdown.map { |cve| cve[:cve_id] } cve_ids_in_db = TrickestPocCve.where(:cve_id => ids).pluck(:cve_id) new_cve_ids = ids - cve_ids_in_db - new_cves = cves_from_json.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } + new_cves = cves_from_markdown.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } puts "Importing any new CVEs from #{year}" bulk_insert(new_cves)