add support for trickest poc cve github repo

.gitignore

@@ -32,3 +32,6 @@
 
 # Any API keys or envars we dont want to commit add here. 
 /twitter_credentials.env
+
+# Ignore our data dir as that gets populated during initial seed/setup
+/data/*

README.md

@@ -3,10 +3,11 @@
 Import common security data such as CVE, CPE, and Security Advisories from various CNAs into a rails app with a postgresql db backend.
 
 ## Supported data models:
-- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) mitre.
+- `Cve` data from [cve_list](https://github.com/CVEProject/cvelist) github repo.
 - `Cpe` data from [nvd](https://nvd.nist.gov/products/cpe) 2.2 format.
-- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub).
+- `GithubPoc` data from [nomi-sec](https://github.com/nomi-sec/PoC-in-GitHub) github repo.
 - `InthewildCveExploit` data from [inthewild.io](https://inthewild.io/api/exploited) exploited feed.
+- `TrickestPocCve` data from [trickest](https://github.com/trickest/cve) github repo.
 
 ## Initial Setup
 
@@ -43,10 +44,19 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
   get "/github_pocs", to: "github_pocs#index"
   get "/github_pocs/:id", to: "github_pocs#show"
   get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve"
+  get "/github_pocs/years/:year", to: "github_pocs#show_year"
 ```
 
 #### InthewildCveExploits
 ```
   get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index"
   get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show"
+```
+
+#### TrickestPocCves
+```
+  get "/trickest_poc_cves", to: "trickest_poc_cves#index"
+  get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show"
+  get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve"
+  get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year"
 ```

app/controllers/github_pocs_controller.rb

@@ -13,4 +13,9 @@ class GithubPocsController < ApplicationController
     render json: @poc.to_json
   end
 
+  def show_year
+    @cves_for_year = GithubPoc.from_year(params[:year])
+    render json: @cves_for_year.to_json
+  end
+
 end

app/controllers/trickest_poc_cves_controller.rb

@@ -0,0 +1,21 @@
+class TrickestPocCvesController < ApplicationController
+  def index
+    @pocs = TrickestPocCve.all
+  end
+
+  def show
+    @poc = TrickestPocCve.find_by(:id => params[:id])
+    render json: @poc.to_json
+  end
+
+  def show_for_cve
+    @poc = TrickestPocCve.where(:cve_id => params[:cve_id])
+    render json: @poc.to_json
+  end
+
+  def show_year
+    @cves_for_year = TrickestPocCve.from_year(params[:year])
+    render json: @cves_for_year.to_json
+  end
+
+end

app/models/github_poc.rb

@@ -1,2 +1,5 @@
 class GithubPoc < ActiveRecord::Base
+    def self.from_year(year)
+        where("cve_id LIKE ?", "CVE-#{year}-%")
+    end
 end

app/models/trickest_poc_cve.rb

@@ -0,0 +1,5 @@
+class TrickestPocCve < ActiveRecord::Base
+    def self.from_year(year)
+        where("cve_id LIKE ?", "CVE-#{year}-%")
+    end
+end

app/views/trickest_poc_cves/index.html.erb

@@ -0,0 +1 @@
+<h1>TrickestPocCves#index</h1>

app/views/trickest_poc_cves/show.html.erb

@@ -0,0 +1,2 @@
+<h1> @poc </h1>
+

config/routes.rb

@@ -10,7 +10,14 @@ Rails.application.routes.draw do
   get "/github_pocs", to: "github_pocs#index"
   get "/github_pocs/:id", to: "github_pocs#show"
   get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve"
+  get "/github_pocs/years/:year", to: "github_pocs#show_year"
 
   get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index"
   get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show"
+
+  get "/trickest_poc_cves", to: "trickest_poc_cves#index"
+  get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show"
+  get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve"
+  get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year"
+
 end

db/migrate/20220407051821_create_trickest_poc_cves.rb

@@ -0,0 +1,10 @@
+class CreateTrickestPocCves < ActiveRecord::Migration[7.0]
+  def change
+    create_table :trickest_poc_cves do |t|
+      t.string :cve_id
+      t.string :cve_url
+      t.string :description
+      t.string :poc_links, array: true
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_07_051821) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -69,4 +69,11 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_06_064613) do
     t.string "earliest_report"
   end
 
+  create_table "trickest_poc_cves", force: :cascade do |t|
+    t.string "cve_id"
+    t.string "cve_url"
+    t.string "description"
+    t.string "poc_links", array: true
+  end
+
 end

db/seeds.rb

@@ -10,6 +10,7 @@ require '/data_importer/lib/cpe_importer.rb'
 require '/data_importer/lib/cve_list_importer.rb'
 require '/data_importer/lib/poc_in_github_importer.rb'
 require '/data_importer/lib/inthewild_cve_exploit_importer.rb'
+require '/data_importer/lib/trickest_poc_cve_importer.rb'
 
 def line_sep
   puts '----------' * 12
@@ -18,6 +19,7 @@ end
 def perform
     import_cves
     import_github_pocs
+    import_trickest_poc_cves
     import_inthewild_cve_exploits
     import_cpes
 end
@@ -42,4 +44,9 @@ def import_inthewild_cve_exploits
     InthewildCveExploitImporter.new.import
 end
 
+def import_trickest_poc_cves
+    line_sep
+    TrickestPocCveImporter.new.import
+end
+
 perform

lib/cve_list_importer.rb

@@ -72,7 +72,8 @@ class CveListImporter
       git_clone_repo
     end
 
-    puts "Now starting import for CveList."
+    puts "Now starting import for #{repo_url}."
+    puts '----------' * 12
     (1999..Date.today.year).map do |year|
       cves_from_json = cves_for_year(year)
 

lib/poc_in_github_importer.rb

@@ -106,7 +106,7 @@ class PocInGithubImporter
       git_clone_repo
     end
 
-    puts "Now starting import for PocInGithub."
+    puts "Now starting import for #{repo_url}."
     (1999..Date.today.year).map do |year|
       cves_from_json = cves_for_year(year)
 

lib/trickest_poc_cve_importer.rb

@@ -50,15 +50,12 @@ class TrickestPocCveImporter
         p_text = doc.xpath('//p').map {|p| p.text }
         links_for_poc = doc.xpath('//p/a').map {|a| a.values}.flatten
         
-        data_hash["#{cve_id}"] = cve_url 
+        data_hash['cve_id'] = cve_id 
+        data_hash['cve_url'] = cve_url
         # p_text[0] is always an ' '.
-        data_hash['Description'] = p_text[1]
+        data_hash['description'] = p_text[1]
 
-        # array of values if its a links. hard to distinguish between ones under POC and ones under Github
-        # if it contains no data under the heading there will be no .value but instead .text will return data.
-        # these ones can both have multiple values
-        # just normalize and put POC and Github stuff under one key now. idc i just need the URL
-        data_hash['POC'] = links_for_poc  
+        data_hash['poc_links'] = links_for_poc  
         data_hash
     end
   
@@ -80,25 +77,19 @@ class TrickestPocCveImporter
   
     def cve_attrs_from_item(json)
       cve_attrs = {}
-      #cve_attrs[:cve_data_meta] = json['CVE_data_meta']
-      #cve_attrs[:cve_id] = json['CVE_data_meta']['ID']
-      #cve_attrs[:affects] = json['affects']
-      #cve_attrs[:data_format] = json['data_format']
-      #cve_attrs[:data_type] = json['data_type']
-      #cve_attrs[:data_version] = json['data_version']
-      #cve_attrs[:description] = json['description']
-      #cve_attrs[:impact] = json['impact']
-      #cve_attrs[:problemtype] = json['problemtype']
-      #cve_attrs[:references] = json['references']
-      #cve_attrs[:source] = json['source']
+      cve_attrs[:cve_id] = json['cve_id']
+      cve_attrs[:cve_url] = json['cve_url'] 
+      cve_attrs[:description] = json['description']
+      cve_attrs[:poc_links] = json['poc_links']
       cve_attrs
     end
   
     # for bulk inserting
     def cves_for_year(year)
-      json_data = read_jsons_for_year(year)
-      json_data.map do |json_f|
-        cve_attrs_from_item(json_f)
+      htmls = read_mds_for_year(year)
+      htmls.map do |html|
+        data_hash = html_to_hash(html)
+        cve_attrs_from_item(data_hash)
       end
     end
   
@@ -109,15 +100,16 @@ class TrickestPocCveImporter
         git_clone_repo
       end
   
-      puts "Now starting import for CveList."
+      puts "Now starting import for #{repo_url}."
+      puts '----------' * 12
       (1999..Date.today.year).map do |year|
-        cves_from_json = cves_for_year(year)
+        cves_from_markdown = cves_for_year(year)
   
-        ids = cves_from_json.map { |cve| cve[:cve_id] }
+        ids = cves_from_markdown.map { |cve| cve[:cve_id] }
         cve_ids_in_db = TrickestPocCve.where(:cve_id => ids).pluck(:cve_id)
   
         new_cve_ids = ids - cve_ids_in_db
-        new_cves = cves_from_json.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) }
+        new_cves = cves_from_markdown.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) }
         puts "Importing any new CVEs from #{year}"
   
         bulk_insert(new_cves)