From 8cbe59f55b7c67a3f3b91e407283c98eb6f5ee93 Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Mon, 4 Apr 2022 13:18:03 -0500 Subject: [PATCH] make json rendering for cpes and cves --- Gemfile | 1 + Gemfile.lock | 22 +++- app/controllers/cpes_controller.rb | 10 ++ app/controllers/cves_controller.rb | 14 +++ app/helpers/cpes_helper.rb | 2 + app/helpers/cves_helper.rb | 2 + app/models/cpe.rb | 1 + app/views/cpes/index.html.erb | 1 + app/views/cpes/show.html.erb | 8 ++ app/views/cves/index.html.erb | 1 + app/views/cves/show.html.erb | 12 +++ bin/docker_database_setup.sh | 6 ++ bin/docker_rebuild.sh | 2 + config/routes.rb | 6 ++ db/migrate/20220404150811_create_cpes.rb | 13 +++ db/schema.rb | 12 ++- db/seeds.rb | 8 +- lib/cna_security_advisories.rb | 42 ++++++++ lib/cpe_importer.rb | 131 +++++++++++++++++++++++ test/controllers/cpes_controller_test.rb | 7 ++ test/controllers/cves_controller_test.rb | 7 ++ 21 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 app/controllers/cpes_controller.rb create mode 100644 app/controllers/cves_controller.rb create mode 100644 app/helpers/cpes_helper.rb create mode 100644 app/helpers/cves_helper.rb create mode 100644 app/models/cpe.rb create mode 100644 app/views/cpes/index.html.erb create mode 100644 app/views/cpes/show.html.erb create mode 100644 app/views/cves/index.html.erb create mode 100644 app/views/cves/show.html.erb create mode 100755 bin/docker_database_setup.sh create mode 100644 db/migrate/20220404150811_create_cpes.rb create mode 100644 lib/cna_security_advisories.rb create mode 100644 lib/cpe_importer.rb create mode 100644 test/controllers/cpes_controller_test.rb create mode 100644 test/controllers/cves_controller_test.rb diff --git a/Gemfile b/Gemfile index fb58634..4d700f1 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gem 'rails', '~> 7.0.0' gem 'actionpack' gem 'sass-rails' gem 'railties' +gem 'rest-client' # Use postgres as the database for Active Record gem 'pg' diff --git a/Gemfile.lock b/Gemfile.lock index da7d53e..09da9df 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -102,6 +102,8 @@ GEM concurrent-ruby (1.1.10) crass (1.0.6) digest (3.1.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) erubi (1.10.0) execjs (2.8.1) ffi (1.15.5) @@ -109,6 +111,9 @@ GEM rchardet (~> 1.8) globalid (1.0.0) activesupport (>= 5.0) + http-accept (1.7.0) + http-cookie (1.0.4) + domain_name (~> 0.5) i18n (1.10.0) concurrent-ruby (~> 1.0) interception (0.5) @@ -128,7 +133,11 @@ GEM marcel (1.0.2) matrix (0.4.2) method_source (1.0.0) + mime-types (3.4.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2022.0105) mini_mime (1.1.2) + mini_portile2 (2.8.0) minitest (5.15.0) msgpack (1.4.5) net-imap (0.2.3) @@ -145,8 +154,10 @@ GEM digest net-protocol timeout + netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.13.3-x86_64-linux) + nokogiri (1.13.3) + mini_portile2 (~> 2.8.0) racc (~> 1.4) pg (1.3.5) pry (0.13.1) @@ -203,6 +214,11 @@ GEM ffi (~> 1.0) rchardet (1.8.0) regexp_parser (2.2.1) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) rexml (3.2.5) ruby_dep (1.5.0) rubyzip (2.3.2) @@ -242,6 +258,9 @@ GEM concurrent-ruby (~> 1.0) uglifier (4.2.0) execjs (>= 0.3.0, < 3) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.1) web-console (4.2.0) actionview (>= 6.0.0) activemodel (>= 6.0.0) @@ -282,6 +301,7 @@ DEPENDENCIES puma (~> 3.11) rails (~> 7.0.0) railties + rest-client sass-rails selenium-webdriver spring diff --git a/app/controllers/cpes_controller.rb b/app/controllers/cpes_controller.rb new file mode 100644 index 0000000..bbf7d78 --- /dev/null +++ b/app/controllers/cpes_controller.rb @@ -0,0 +1,10 @@ +class CpesController < ApplicationController + def index + @cpes = Cpe.all + end + + def show + @cpe = Cpe.find(params[:id]) + render json: @cpe.to_json + end +end diff --git a/app/controllers/cves_controller.rb b/app/controllers/cves_controller.rb new file mode 100644 index 0000000..70d2e1e --- /dev/null +++ b/app/controllers/cves_controller.rb @@ -0,0 +1,14 @@ +class CvesController < ApplicationController + def index + @cves = Cve.all + end + + def show + @cve = Cve.find_by_id(params[:cve_id]) + render json: @cve.to_json + end + + def show_year + @cves_for_year = Cve.for_year(params[:year]) + end +end diff --git a/app/helpers/cpes_helper.rb b/app/helpers/cpes_helper.rb new file mode 100644 index 0000000..b9a5e55 --- /dev/null +++ b/app/helpers/cpes_helper.rb @@ -0,0 +1,2 @@ +module CpesHelper +end diff --git a/app/helpers/cves_helper.rb b/app/helpers/cves_helper.rb new file mode 100644 index 0000000..c1f2986 --- /dev/null +++ b/app/helpers/cves_helper.rb @@ -0,0 +1,2 @@ +module CvesHelper +end diff --git a/app/models/cpe.rb b/app/models/cpe.rb new file mode 100644 index 0000000..f348b8e --- /dev/null +++ b/app/models/cpe.rb @@ -0,0 +1 @@ +class Cpe < ActiveRecord::Base; end \ No newline at end of file diff --git a/app/views/cpes/index.html.erb b/app/views/cpes/index.html.erb new file mode 100644 index 0000000..643fbd5 --- /dev/null +++ b/app/views/cpes/index.html.erb @@ -0,0 +1 @@ +

Cpes#index

diff --git a/app/views/cpes/show.html.erb b/app/views/cpes/show.html.erb new file mode 100644 index 0000000..c2f310a --- /dev/null +++ b/app/views/cpes/show.html.erb @@ -0,0 +1,8 @@ +

<%= @cpe.id %>

+ +

Status: <%= @cpe.status %>

+

Modifcation Date: <%= @cpe.modification_date %>

+

NVD ID: <%= @cpe.nvd_id %>

+

References: <%= @cpe.references %>

+

Title: <%= @cpe.title %>

+

Name: <%= @cpe.name %>

diff --git a/app/views/cves/index.html.erb b/app/views/cves/index.html.erb new file mode 100644 index 0000000..f59aca7 --- /dev/null +++ b/app/views/cves/index.html.erb @@ -0,0 +1 @@ +

Cves#index

\ No newline at end of file diff --git a/app/views/cves/show.html.erb b/app/views/cves/show.html.erb new file mode 100644 index 0000000..0357be3 --- /dev/null +++ b/app/views/cves/show.html.erb @@ -0,0 +1,12 @@ +

<%= @cve.cve_id %>

+ +

cve_data_meta: <%= @cve.cve_data_meta %>

+

affects: <%= @cve.affects %>

+

data_format: <%= @cve.data_format %>

+

data_type: <%= @cve.data_type %>

+

data_version: <%= @cve.data_version %>

+

description: <%= @cve.description %>

+

impact: <%= @cve.impact %>

+

problemtype: <%= @cve.problemtype %>

+

references: <%= @cve.references %>

+

source: <%= @cve.source %>

diff --git a/bin/docker_database_setup.sh b/bin/docker_database_setup.sh new file mode 100755 index 0000000..96c632b --- /dev/null +++ b/bin/docker_database_setup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# script to run the docker commands cuz docker sux + +docker-compose run web rake db:create +docker-compose run web rake db:migrate +docker-compose run web rake db:setup diff --git a/bin/docker_rebuild.sh b/bin/docker_rebuild.sh index ae7524c..6aa9ffe 100755 --- a/bin/docker_rebuild.sh +++ b/bin/docker_rebuild.sh @@ -2,4 +2,6 @@ # docker rebuild and bundle install # updates Gemfile.lock +docker-compose down +docker-compose build docker-compose run web bundle install diff --git a/config/routes.rb b/config/routes.rb index 787824f..cfefbc3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,9 @@ Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html + get "/cves", to: "cves#index" + get "/cves/:cve_id", to: "cves#show" + get "/cves/:year", to: "cves#show_year" + + get "/cpes", to: "cpes#index" + get "/cpes/:id", to: "cpes#show" end diff --git a/db/migrate/20220404150811_create_cpes.rb b/db/migrate/20220404150811_create_cpes.rb new file mode 100644 index 0000000..92b87e3 --- /dev/null +++ b/db/migrate/20220404150811_create_cpes.rb @@ -0,0 +1,13 @@ +class CreateCpes < ActiveRecord::Migration[7.0] + def change + create_table :cpes do |t| + t.string :status + t.date :modification_date + t.integer :nvd_id + t.index :nvd_id, unique: true + t.jsonb :references + t.string :title + t.string :name + end + end +end diff --git a/db/schema.rb b/db/schema.rb index d0417ed..c264dfa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,20 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_04_01_173431) do +ActiveRecord::Schema[7.0].define(version: 2022_04_04_150811) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "cpes", force: :cascade do |t| + t.string "status" + t.date "modification_date" + t.integer "nvd_id" + t.jsonb "references" + t.string "title" + t.string "name" + t.index ["nvd_id"], name: "index_cpes_on_nvd_id", unique: true + end + create_table "cves", force: :cascade do |t| t.jsonb "cve_data_meta" t.string "cve_id" diff --git a/db/seeds.rb b/db/seeds.rb index 58db0c2..d93bb35 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -6,6 +6,10 @@ # movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }]) # Character.create(name: 'Luke', movie: movies.first) -# this should get any new Cves and create them in the db +require '/data_importer/lib/cpe_importer.rb' require '/data_importer/lib/cve_list_importer.rb' -CveListImporter.new.import \ No newline at end of file + +# this should get any new Cves and create them in the db +CveListImporter.new.import +# this should recreate CPE data +CpeImporter.download_and_import \ No newline at end of file diff --git a/lib/cna_security_advisories.rb b/lib/cna_security_advisories.rb new file mode 100644 index 0000000..38300e8 --- /dev/null +++ b/lib/cna_security_advisories.rb @@ -0,0 +1,42 @@ +# outputs the list of CNA organizationNames and the securityAdvisory urls from the json file here: +# https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json + +require 'json' +require 'rest-client' + +class CnaSecurityAdvisories + attr_accessor :url + def initialize + @url = 'https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json' + end + + def send_request_rest + RestClient::Request.execute( + method: :get, + url: url + ) + end + + def parse_res(response) + JSON.parse(response.body) + end + + def get_json + res = send_request_rest + if res.code == 200 + parse_res(res) + else + "HTTP Status: #{res.code}" + end + end + + def perform + json = get_json + json.map do |d| + org_name = d.dig('organizationName') + security_advisories = d.dig('securityAdvisories') + security_advisory_urls = security_advisories.dig('advisories').map { |adv| adv.dig('url') } + { orgName: org_name, security_advisories_urls: security_advisory_urls } + end + end +end diff --git a/lib/cpe_importer.rb b/lib/cpe_importer.rb new file mode 100644 index 0000000..2b81adc --- /dev/null +++ b/lib/cpe_importer.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require 'bulk_insert' +require 'nokogiri' +require 'net/http' + +# use this to import CPE data into postgres database +class CpeImporter + XML_NAMESPACES = { + 'meta' => 'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2', + 'xsi' => 'http://www.w3.org/2001/XMLSchema-instance', + '' => 'http://cpe.mitre.org/dictionary/2.0' + }.freeze + + # TODO: v2.3 is available, see https://cpe.mitre.org/specification/ + URL = 'https://nvd.nist.gov' \ + '/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz' + + def self.download + ActiveSupport::Notifications.instrument 'downloaded.cpe_importer' do + uri = URI.parse(URL) + Net::HTTP.start(uri.host, uri.port, + use_ssl: uri.scheme == 'https') do |http| + request = Net::HTTP::Get.new uri + http.request request do |response| + if (response.code.to_i < 200) || (response.code.to_i > 299) + raise StandardError, "Bad CPE def request: #{response.code}: #{response.body}" + end + + read_file_chunks(response) + end + end + end + end + + def self.read_file_chunks(response) + File.open('/data_importer/data/official-cpe-dictionary_v2.2.xml.gz', 'w') do |io| + response.read_body do |chunk| + io.write chunk.force_encoding('UTF-8') + end + end + end + + def self.transform_node(node) + Nokogiri::XML(node.outer_xml).root + end + + def self.accept_node(node) + node.name == 'cpe-item' && node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT + end + + def self.import(bulk_count = 20000, filepath = '/data_importer/data/official-cpe-dictionary_v2.2.xml.gz') + Zlib::GzipReader.open(filepath) do |file| + items = [] + Nokogiri::XML::Reader.from_io(file).each do |node| + items << transform_node(node) if accept_node(node) + + if items.count == bulk_count + create_cpes(items) + items = [] + end + end + create_cpes(items) if items.any? + rescue Nokogiri::XML::SyntaxError => e + if file.nil? == false + file.rewind + file_content_sample = file.read(400) + handle_error("Invalid XML in this file: \"#{file_content_sample}\" - original error #{$ERROR_INFO}") + end + + # Couldn't add more info, just re-raise the error + raise e + end + rescue Zlib::GzipFile::Error + handle_error("Unable to decompress cpe dictionary: #{$ERROR_INFO}") + end + + def self.handle_error(error_message) + raise $ERROR_INFO, + error_message.to_s, + $ERROR_INFO.backtrace + end + + def self.create_cpes(items) + cpes = items.map do |item| + cpe_attrs_from_item(item) + end + + Cpe.bulk_insert do |worker| + cpes.each do |attrs| + worker.add(attrs) + end + end + end + + def self.cpe_attrs_from_item(item) + cpe_attrs = {} + + item.search('title').each do |title| + cpe_attrs[:title] = title.inner_text if title.attribute('lang').value == 'en-US' + end + + metadata = item.at_xpath('meta:item-metadata', XML_NAMESPACES) + references = item.search('reference').map { |n| { "#{n.text.gsub(' ', '_').downcase}": n.values } } + cpe_attrs[:references] = references + cpe_attrs[:name] = item['name'] unless item['name'].nil? + cpe_attrs[:modification_date] = metadata['modification-date'] + cpe_attrs[:status] = metadata['status'] + cpe_attrs[:nvd_id] = metadata['nvd-id'] + cpe_attrs + end + + def self.create_cpe(item) + cpe_attrs = cpe_attrs_from_item(item) + cpe = Cpe.where(name: cpe_attrs[:name]).first_or_initialize + return unless cpe.new_record? + + cpe.title = cpe_attrs[:title] + cpe.metadata = cpe_attrs[:metadata] + cpe.references = cpe_attrs[:references] + cpe.modification_date = cpe_attrs[:modification_date] + cpe.status = cpe_attrs[:status] + cpe.nvd_id = cpe_attrs[:nvd_id] + cpe.save + end + + def self.download_and_import + download + import + end +end diff --git a/test/controllers/cpes_controller_test.rb b/test/controllers/cpes_controller_test.rb new file mode 100644 index 0000000..ef52cd7 --- /dev/null +++ b/test/controllers/cpes_controller_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class CpesControllerTest < ActionDispatch::IntegrationTest + # test "the truth" do + # assert true + # end +end diff --git a/test/controllers/cves_controller_test.rb b/test/controllers/cves_controller_test.rb new file mode 100644 index 0000000..e03181e --- /dev/null +++ b/test/controllers/cves_controller_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class CvesControllerTest < ActionDispatch::IntegrationTest + # test "the truth" do + # assert true + # end +end