From 2d729a6999b52cd8489b3830e0889e3f8a2f3250 Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Thu, 7 Apr 2022 04:32:08 -0500 Subject: [PATCH] added cvemon data support --- README.md | 8 ++++ app/controllers/cvemon_cves_controller.rb | 21 ++++++++++ app/models/cvemon_cve.rb | 5 +++ app/views/cvemon_cves/index.html.erb | 1 + app/views/cvemon_cves/show.html.erb | 2 + config/routes.rb | 5 +++ .../20220407083218_create_cvemon_cves.rb | 9 ++++ db/schema.rb | 8 +++- db/seeds.rb | 7 ++++ lib/cvemon_cve_importer.rb | 41 +++++++++++++++++++ lib/poc_in_github_importer.rb | 1 + 11 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 app/controllers/cvemon_cves_controller.rb create mode 100644 app/models/cvemon_cve.rb create mode 100644 app/views/cvemon_cves/index.html.erb create mode 100644 app/views/cvemon_cves/show.html.erb create mode 100644 db/migrate/20220407083218_create_cvemon_cves.rb create mode 100644 lib/cvemon_cve_importer.rb diff --git a/README.md b/README.md index 30152e1..b6c353a 100644 --- a/README.md +++ b/README.md @@ -59,4 +59,12 @@ For now unauthenticated api over localhost:3000 until I put in some basic token get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show" get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve" get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year" +``` + +#### CvemonCves +``` + get "/cvemon_cves", to: "cvemon_cves#index" + get "/cvemon_cves/:id", to: "cvemon_cves#show" + get "/cvemon_cves/cve/:cve_id", to: "cvemon_cves#show_for_cve" + get "/cvemon_cves/years/:year", to: "cvemon_cves#show_year" ``` \ No newline at end of file diff --git a/app/controllers/cvemon_cves_controller.rb b/app/controllers/cvemon_cves_controller.rb new file mode 100644 index 0000000..db19db4 --- /dev/null +++ b/app/controllers/cvemon_cves_controller.rb @@ -0,0 +1,21 @@ +class CvemonCvesController < ApplicationController + def index + @pocs = CvemonCve.all + end + + def show + @poc = CvemonCve.find_by(:id => params[:id]) + render json: @poc.to_json + end + + def show_for_cve + @poc = CvemonCve.where(:cve_id => params[:cve_id]) + render json: @poc.to_json + end + + def show_year + @cves_for_year = CvemonCve.from_year(params[:year]) + render json: @cves_for_year.to_json + end + +end diff --git a/app/models/cvemon_cve.rb b/app/models/cvemon_cve.rb new file mode 100644 index 0000000..d7a4931 --- /dev/null +++ b/app/models/cvemon_cve.rb @@ -0,0 +1,5 @@ +class CvemonCve < ActiveRecord::Base + def self.from_year(year) + where("cve_id LIKE ?", "CVE-#{year}-%") + end +end diff --git a/app/views/cvemon_cves/index.html.erb b/app/views/cvemon_cves/index.html.erb new file mode 100644 index 0000000..4bcdd46 --- /dev/null +++ b/app/views/cvemon_cves/index.html.erb @@ -0,0 +1 @@ +

CvemonCves#index

diff --git a/app/views/cvemon_cves/show.html.erb b/app/views/cvemon_cves/show.html.erb new file mode 100644 index 0000000..740ff12 --- /dev/null +++ b/app/views/cvemon_cves/show.html.erb @@ -0,0 +1,2 @@ +

@poc

+ diff --git a/config/routes.rb b/config/routes.rb index 24b1d34..e4e3e49 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -20,4 +20,9 @@ Rails.application.routes.draw do get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve" get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year" + get "/cvemon_cves", to: "cvemon_cves#index" + get "/cvemon_cves/:id", to: "cvemon_cves#show" + get "/cvemon_cves/cve/:cve_id", to: "cvemon_cves#show_for_cve" + get "/cvemon_cves/years/:year", to: "cvemon_cves#show_year" + end diff --git a/db/migrate/20220407083218_create_cvemon_cves.rb b/db/migrate/20220407083218_create_cvemon_cves.rb new file mode 100644 index 0000000..7d76695 --- /dev/null +++ b/db/migrate/20220407083218_create_cvemon_cves.rb @@ -0,0 +1,9 @@ +class CreateCvemonCves < ActiveRecord::Migration[7.0] + def change + create_table :cvemon_cves do |t| + t.string :cve_id + t.index :cve_id, unique: true + t.string :urls, array: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1c8580f..84da295 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_04_07_051821) do +ActiveRecord::Schema[7.0].define(version: 2022_04_07_083218) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -24,6 +24,12 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_07_051821) do t.index ["nvd_id"], name: "index_cpes_on_nvd_id", unique: true end + create_table "cvemon_cves", force: :cascade do |t| + t.string "cve_id" + t.string "urls", array: true + t.index ["cve_id"], name: "index_cvemon_cves_on_cve_id", unique: true + end + create_table "cves", force: :cascade do |t| t.jsonb "cve_data_meta" t.string "cve_id" diff --git a/db/seeds.rb b/db/seeds.rb index db6c918..8f0ab18 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -11,6 +11,7 @@ require '/data_importer/lib/cve_list_importer.rb' require '/data_importer/lib/poc_in_github_importer.rb' require '/data_importer/lib/inthewild_cve_exploit_importer.rb' require '/data_importer/lib/trickest_poc_cve_importer.rb' +require '/data_importer/lib/cvemon_cve_importer.rb' def line_sep puts '----------' * 12 @@ -21,6 +22,7 @@ def perform import_github_pocs import_trickest_poc_cves import_inthewild_cve_exploits + import_cvemon_cves import_cpes end @@ -49,4 +51,9 @@ def import_trickest_poc_cves TrickestPocCveImporter.new.import end +def import_cvemon_cves + line_sep + CvemonCveImporter.new.import +end + perform \ No newline at end of file diff --git a/lib/cvemon_cve_importer.rb b/lib/cvemon_cve_importer.rb new file mode 100644 index 0000000..5d1e7ee --- /dev/null +++ b/lib/cvemon_cve_importer.rb @@ -0,0 +1,41 @@ +require 'rest-client' +require 'json' + +class CvemonCveImporter + attr_accessor :url + def initialize + @url = 'https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json' + end + + def get_cve_data + r = RestClient::Request.execute( + :method => :get, + :url => url, + :headers => {"Content-type": "application/json"} + ) + if r.code == 200 + JSON.parse(r.body) + else + puts "HTTP Code #{r.code}" + end + end + + def bulk_insert(cves) + CvemonCve.bulk_insert do |worker| + cves.each do |attrs| + worker.add(attrs) + end + end + end + + def import + feed = get_cve_data + cve_ids = feed.keys + puts "Now importing CvemonCves." + cves = cve_ids.map do |cve_id| + { :cve_id => cve_id, :urls => feed[cve_id] } + end + + bulk_insert(cves) + end +end \ No newline at end of file diff --git a/lib/poc_in_github_importer.rb b/lib/poc_in_github_importer.rb index bf7a1c8..037bbf8 100644 --- a/lib/poc_in_github_importer.rb +++ b/lib/poc_in_github_importer.rb @@ -107,6 +107,7 @@ class PocInGithubImporter end puts "Now starting import for #{repo_url}." + puts '----------' * 12 (1999..Date.today.year).map do |year| cves_from_json = cves_for_year(year)