diff --git a/.gitignore b/.gitignore
index 9e1b5ec..ecca9c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,6 @@
# Any API keys or envars we dont want to commit add here.
/twitter_credentials.env
+
+# Ignore our data dir as that gets populated during initial seed/setup
+/data/*
diff --git a/README.md b/README.md
index 1f3aa23..30152e1 100644
--- a/README.md
+++ b/README.md
@@ -3,10 +3,11 @@
Import common security data such as CVE, CPE, and Security Advisories from various CNAs into a rails app with a postgresql db backend.
## Supported data models:
-- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) mitre.
+- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) github repo.
- `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format.
-- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub).
+- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo.
- `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed.
+- `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo.
## Initial Setup
@@ -43,10 +44,19 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
get "/github_pocs", to: "github_pocs#index"
get "/github_pocs/:id", to: "github_pocs#show"
get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve"
+ get "/github_pocs/years/:year", to: "github_pocs#show_year"
```
#### InthewildCveExploits
```
get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index"
get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show"
+```
+
+#### TrickestPocCves
+```
+ get "/trickest_poc_cves", to: "trickest_poc_cves#index"
+ get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show"
+ get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve"
+ get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year"
```
\ No newline at end of file
diff --git a/app/controllers/github_pocs_controller.rb b/app/controllers/github_pocs_controller.rb
index 79861ca..3f11751 100644
--- a/app/controllers/github_pocs_controller.rb
+++ b/app/controllers/github_pocs_controller.rb
@@ -13,4 +13,9 @@ class GithubPocsController < ApplicationController
render json: @poc.to_json
end
+ def show_year
+ @cves_for_year = GithubPoc.from_year(params[:year])
+ render json: @cves_for_year.to_json
+ end
+
end
diff --git a/app/controllers/trickest_poc_cves_controller.rb b/app/controllers/trickest_poc_cves_controller.rb
new file mode 100644
index 0000000..1e0b876
--- /dev/null
+++ b/app/controllers/trickest_poc_cves_controller.rb
@@ -0,0 +1,21 @@
+class TrickestPocCvesController < ApplicationController
+ def index
+ @pocs = TrickestPocCve.all
+ end
+
+ def show
+ @poc = TrickestPocCve.find_by(:id => params[:id])
+ render json: @poc.to_json
+ end
+
+ def show_for_cve
+ @poc = TrickestPocCve.where(:cve_id => params[:cve_id])
+ render json: @poc.to_json
+ end
+
+ def show_year
+ @cves_for_year = TrickestPocCve.from_year(params[:year])
+ render json: @cves_for_year.to_json
+ end
+
+end
diff --git a/app/models/github_poc.rb b/app/models/github_poc.rb
index 52d763c..f804d89 100644
--- a/app/models/github_poc.rb
+++ b/app/models/github_poc.rb
@@ -1,2 +1,5 @@
class GithubPoc < ActiveRecord::Base
+ def self.from_year(year)
+ where("cve_id LIKE ?", "CVE-#{year}-%")
+ end
end
diff --git a/app/models/trickest_poc_cve.rb b/app/models/trickest_poc_cve.rb
new file mode 100644
index 0000000..4aa7d69
--- /dev/null
+++ b/app/models/trickest_poc_cve.rb
@@ -0,0 +1,5 @@
+class TrickestPocCve < ActiveRecord::Base
+ def self.from_year(year)
+ where("cve_id LIKE ?", "CVE-#{year}-%")
+ end
+end
diff --git a/app/views/trickest_poc_cves/index.html.erb b/app/views/trickest_poc_cves/index.html.erb
new file mode 100644
index 0000000..8874e23
--- /dev/null
+++ b/app/views/trickest_poc_cves/index.html.erb
@@ -0,0 +1 @@
+
TrickestPocCves#index
diff --git a/app/views/trickest_poc_cves/show.html.erb b/app/views/trickest_poc_cves/show.html.erb
new file mode 100644
index 0000000..740ff12
--- /dev/null
+++ b/app/views/trickest_poc_cves/show.html.erb
@@ -0,0 +1,2 @@
+ @poc
+
diff --git a/config/routes.rb b/config/routes.rb
index 0ed9775..24b1d34 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -10,7 +10,14 @@ Rails.application.routes.draw do
get "/github_pocs", to: "github_pocs#index"
get "/github_pocs/:id", to: "github_pocs#show"
get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve"
+ get "/github_pocs/years/:year", to: "github_pocs#show_year"
get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index"
get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show"
+
+ get "/trickest_poc_cves", to: "trickest_poc_cves#index"
+ get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show"
+ get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve"
+ get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year"
+
end
diff --git a/db/migrate/20220407051821_create_trickest_poc_cves.rb b/db/migrate/20220407051821_create_trickest_poc_cves.rb
new file mode 100644
index 0000000..8fda9fd
--- /dev/null
+++ b/db/migrate/20220407051821_create_trickest_poc_cves.rb
@@ -0,0 +1,10 @@
+class CreateTrickestPocCves < ActiveRecord::Migration[7.0]
+ def change
+ create_table :trickest_poc_cves do |t|
+ t.string :cve_id
+ t.string :cve_url
+ t.string :description
+ t.string :poc_links, array: true
+ end
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 972c76e..1c8580f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_07_051821) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -69,4 +69,11 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do
t.string "earliest_report"
end
+ create_table "trickest_poc_cves", force: :cascade do |t|
+ t.string "cve_id"
+ t.string "cve_url"
+ t.string "description"
+ t.string "poc_links", array: true
+ end
+
end
diff --git a/db/seeds.rb b/db/seeds.rb
index 6598da4..db6c918 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -10,6 +10,7 @@ require '/data_importer/lib/cpe_importer.rb'
require '/data_importer/lib/cve_list_importer.rb'
require '/data_importer/lib/poc_in_github_importer.rb'
require '/data_importer/lib/inthewild_cve_exploit_importer.rb'
+require '/data_importer/lib/trickest_poc_cve_importer.rb'
def line_sep
puts '----------' * 12
@@ -18,6 +19,7 @@ end
def perform
import_cves
import_github_pocs
+ import_trickest_poc_cves
import_inthewild_cve_exploits
import_cpes
end
@@ -42,4 +44,9 @@ def import_inthewild_cve_exploits
InthewildCveExploitImporter.new.import
end
+def import_trickest_poc_cves
+ line_sep
+ TrickestPocCveImporter.new.import
+end
+
perform
\ No newline at end of file
diff --git a/lib/cve_list_importer.rb b/lib/cve_list_importer.rb
index f9cc64f..7889769 100644
--- a/lib/cve_list_importer.rb
+++ b/lib/cve_list_importer.rb
@@ -72,7 +72,8 @@ class CveListImporter
git_clone_repo
end
- puts "Now starting import for CveList."
+ puts "Now starting import for #{repo_url}."
+ puts '----------' * 12
(1999..Date.today.year).map do |year|
cves_from_json = cves_for_year(year)
diff --git a/lib/poc_in_github_importer.rb b/lib/poc_in_github_importer.rb
index 5887cfe..bf7a1c8 100644
--- a/lib/poc_in_github_importer.rb
+++ b/lib/poc_in_github_importer.rb
@@ -106,7 +106,7 @@ class PocInGithubImporter
git_clone_repo
end
- puts "Now starting import for PocInGithub."
+ puts "Now starting import for #{repo_url}."
(1999..Date.today.year).map do |year|
cves_from_json = cves_for_year(year)
diff --git a/lib/trickest_poc_cve_importer.rb b/lib/trickest_poc_cve_importer.rb
index 3bcfe38..8f5d58b 100644
--- a/lib/trickest_poc_cve_importer.rb
+++ b/lib/trickest_poc_cve_importer.rb
@@ -50,15 +50,12 @@ class TrickestPocCveImporter
p_text = doc.xpath('//p').map {|p| p.text }
links_for_poc = doc.xpath('//p/a').map {|a| a.values}.flatten
- data_hash["#{cve_id}"] = cve_url
+ data_hash['cve_id'] = cve_id
+ data_hash['cve_url'] = cve_url
# p_text[0] is always an ' '.
- data_hash['Description'] = p_text[1]
+ data_hash['description'] = p_text[1]
- # array of values if its a links. hard to distinguish between ones under POC and ones under Github
- # if it contains no data under the heading there will be no .value but instead .text will return data.
- # these ones can both have multiple values
- # just normalize and put POC and Github stuff under one key now. idc i just need the URL
- data_hash['POC'] = links_for_poc
+ data_hash['poc_links'] = links_for_poc
data_hash
end
@@ -80,25 +77,19 @@ class TrickestPocCveImporter
def cve_attrs_from_item(json)
cve_attrs = {}
- #cve_attrs[:cve_data_meta] = json['CVE_data_meta']
- #cve_attrs[:cve_id] = json['CVE_data_meta']['ID']
- #cve_attrs[:affects] = json['affects']
- #cve_attrs[:data_format] = json['data_format']
- #cve_attrs[:data_type] = json['data_type']
- #cve_attrs[:data_version] = json['data_version']
- #cve_attrs[:description] = json['description']
- #cve_attrs[:impact] = json['impact']
- #cve_attrs[:problemtype] = json['problemtype']
- #cve_attrs[:references] = json['references']
- #cve_attrs[:source] = json['source']
+ cve_attrs[:cve_id] = json['cve_id']
+ cve_attrs[:cve_url] = json['cve_url']
+ cve_attrs[:description] = json['description']
+ cve_attrs[:poc_links] = json['poc_links']
cve_attrs
end
# for bulk inserting
def cves_for_year(year)
- json_data = read_jsons_for_year(year)
- json_data.map do |json_f|
- cve_attrs_from_item(json_f)
+ htmls = read_mds_for_year(year)
+ htmls.map do |html|
+ data_hash = html_to_hash(html)
+ cve_attrs_from_item(data_hash)
end
end
@@ -109,15 +100,16 @@ class TrickestPocCveImporter
git_clone_repo
end
- puts "Now starting import for CveList."
+ puts "Now starting import for #{repo_url}."
+ puts '----------' * 12
(1999..Date.today.year).map do |year|
- cves_from_json = cves_for_year(year)
+ cves_from_markdown = cves_for_year(year)
- ids = cves_from_json.map { |cve| cve[:cve_id] }
+ ids = cves_from_markdown.map { |cve| cve[:cve_id] }
cve_ids_in_db = TrickestPocCve.where(:cve_id => ids).pluck(:cve_id)
new_cve_ids = ids - cve_ids_in_db
- new_cves = cves_from_json.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) }
+ new_cves = cves_from_markdown.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) }
puts "Importing any new CVEs from #{year}"
bulk_insert(new_cves)