make json rendering for cpes and cves

Gemfile

@@ -8,6 +8,7 @@ gem 'rails', '~> 7.0.0'
 gem 'actionpack' 
 gem 'sass-rails' 
 gem 'railties'
+gem 'rest-client'
 
 # Use postgres as the database for Active Record
 gem 'pg'

Gemfile.lock

@@ -102,6 +102,8 @@ GEM
     concurrent-ruby (1.1.10)
     crass (1.0.6)
     digest (3.1.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
     erubi (1.10.0)
     execjs (2.8.1)
     ffi (1.15.5)
@@ -109,6 +111,9 @@ GEM
       rchardet (~> 1.8)
     globalid (1.0.0)
       activesupport (>= 5.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     interception (0.5)
@@ -128,7 +133,11 @@ GEM
     marcel (1.0.2)
     matrix (0.4.2)
     method_source (1.0.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
     msgpack (1.4.5)
     net-imap (0.2.3)
@@ -145,8 +154,10 @@ GEM
       digest
       net-protocol
       timeout
+    netrc (0.11.0)
     nio4r (2.5.8)
-    nokogiri (1.13.3-x86_64-linux)
+    nokogiri (1.13.3)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     pg (1.3.5)
     pry (0.13.1)
@@ -203,6 +214,11 @@ GEM
       ffi (~> 1.0)
     rchardet (1.8.0)
     regexp_parser (2.2.1)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     rexml (3.2.5)
     ruby_dep (1.5.0)
     rubyzip (2.3.2)
@@ -242,6 +258,9 @@ GEM
       concurrent-ruby (~> 1.0)
     uglifier (4.2.0)
       execjs (>= 0.3.0, < 3)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.1)
     web-console (4.2.0)
       actionview (>= 6.0.0)
       activemodel (>= 6.0.0)
@@ -282,6 +301,7 @@ DEPENDENCIES
   puma (~> 3.11)
   rails (~> 7.0.0)
   railties
+  rest-client
   sass-rails
   selenium-webdriver
   spring

app/controllers/cpes_controller.rb

@@ -0,0 +1,10 @@
+class CpesController < ApplicationController
+  def index
+    @cpes = Cpe.all
+  end
+
+  def show
+    @cpe = Cpe.find(params[:id])
+    render json: @cpe.to_json
+  end
+end

app/controllers/cves_controller.rb

@@ -0,0 +1,14 @@
+class CvesController < ApplicationController
+  def index
+    @cves = Cve.all
+  end
+
+  def show
+    @cve = Cve.find_by_id(params[:cve_id])
+    render json: @cve.to_json
+  end
+
+  def show_year
+    @cves_for_year = Cve.for_year(params[:year])
+  end
+end

app/helpers/cpes_helper.rb

@@ -0,0 +1,2 @@
+module CpesHelper
+end

app/helpers/cves_helper.rb

@@ -0,0 +1,2 @@
+module CvesHelper
+end

app/models/cpe.rb

@@ -0,0 +1 @@
+class Cpe < ActiveRecord::Base; end

app/views/cpes/index.html.erb

@@ -0,0 +1 @@
+<h1>Cpes#index</h1>

app/views/cpes/show.html.erb

@@ -0,0 +1,8 @@
+<h1> <%= @cpe.id %> </h1>
+
+<p> Status: <%= @cpe.status %> </p>
+<p> Modifcation Date: <%= @cpe.modification_date %> </p>
+<p> NVD ID: <%= @cpe.nvd_id %> </p>
+<p> References: <%= @cpe.references %> </p>
+<p> Title: <%= @cpe.title %> </p>
+<p> Name: <%= @cpe.name %> </p>

app/views/cves/index.html.erb

@@ -0,0 +1 @@
+<h1>Cves#index</h1>

app/views/cves/show.html.erb

@@ -0,0 +1,12 @@
+<h1> <%= @cve.cve_id %> </h1>
+
+<p> cve_data_meta: <%= @cve.cve_data_meta %> </p>
+<p> affects: <%= @cve.affects %> </p>
+<p> data_format: <%= @cve.data_format %> </p>
+<p> data_type: <%= @cve.data_type %> </p>
+<p> data_version: <%= @cve.data_version %> </p>
+<p> description: <%= @cve.description %> </p>
+<p> impact: <%= @cve.impact %> </p>
+<p> problemtype: <%= @cve.problemtype %> </p>
+<p> references: <%= @cve.references %> </p>
+<p> source: <%= @cve.source %> </p>

bin/docker_database_setup.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# script to run the docker commands cuz docker sux
+
+docker-compose run web rake db:create
+docker-compose run web rake db:migrate
+docker-compose run web rake db:setup

bin/docker_rebuild.sh

@@ -2,4 +2,6 @@
 # docker rebuild and bundle install
 # updates Gemfile.lock
 
+docker-compose down
+docker-compose build
 docker-compose run web bundle install

config/routes.rb

@@ -1,3 +1,9 @@
 Rails.application.routes.draw do
   # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html
+  get "/cves", to: "cves#index"
+  get "/cves/:cve_id", to: "cves#show"
+  get "/cves/:year", to: "cves#show_year"
+
+  get "/cpes", to: "cpes#index"
+  get "/cpes/:id", to: "cpes#show"
 end

db/migrate/20220404150811_create_cpes.rb

@@ -0,0 +1,13 @@
+class CreateCpes < ActiveRecord::Migration[7.0]
+  def change
+    create_table :cpes do |t|
+      t.string :status
+      t.date :modification_date
+      t.integer :nvd_id
+      t.index :nvd_id, unique: true
+      t.jsonb :references
+      t.string :title
+      t.string :name
+    end
+  end
+end

db/schema.rb

@@ -10,10 +10,20 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_04_01_173431) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_04_150811) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "cpes", force: :cascade do |t|
+    t.string "status"
+    t.date "modification_date"
+    t.integer "nvd_id"
+    t.jsonb "references"
+    t.string "title"
+    t.string "name"
+    t.index ["nvd_id"], name: "index_cpes_on_nvd_id", unique: true
+  end
+
   create_table "cves", force: :cascade do |t|
     t.jsonb "cve_data_meta"
     t.string "cve_id"

db/seeds.rb

@@ -6,6 +6,10 @@
 #   movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
 #   Character.create(name: 'Luke', movie: movies.first)
 
-# this should get any new Cves and create them in the db
+require '/data_importer/lib/cpe_importer.rb'
 require '/data_importer/lib/cve_list_importer.rb'
-CveListImporter.new.import
+
+# this should get any new Cves and create them in the db
+CveListImporter.new.import
+# this should recreate CPE data
+CpeImporter.download_and_import

lib/cna_security_advisories.rb

@@ -0,0 +1,42 @@
+# outputs the list of CNA organizationNames and the securityAdvisory urls from the json file here:
+# https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json
+
+require 'json'
+require 'rest-client'
+
+class CnaSecurityAdvisories 
+  attr_accessor :url
+  def initialize
+    @url = 'https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json'
+  end
+
+  def send_request_rest
+    RestClient::Request.execute(
+      method: :get,
+      url: url
+    )
+  end
+
+  def parse_res(response)
+    JSON.parse(response.body)
+  end
+
+  def get_json
+    res = send_request_rest
+    if res.code == 200
+      parse_res(res)
+    else
+      "HTTP Status: #{res.code}"
+    end
+  end
+
+  def perform
+    json = get_json
+    json.map do |d|
+      org_name = d.dig('organizationName')
+      security_advisories = d.dig('securityAdvisories')
+      security_advisory_urls = security_advisories.dig('advisories').map { |adv| adv.dig('url') }
+      { orgName: org_name, security_advisories_urls: security_advisory_urls }
+    end
+  end
+end

lib/cpe_importer.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'bulk_insert'
+require 'nokogiri'
+require 'net/http'
+
+# use this to import CPE data into postgres database
+class CpeImporter
+  XML_NAMESPACES = {
+    'meta' => 'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2',
+    'xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
+    '' => 'http://cpe.mitre.org/dictionary/2.0'
+  }.freeze
+
+  # TODO: v2.3 is available, see https://cpe.mitre.org/specification/
+  URL = 'https://nvd.nist.gov' \
+        '/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz'
+
+  def self.download
+    ActiveSupport::Notifications.instrument 'downloaded.cpe_importer' do
+      uri = URI.parse(URL)
+      Net::HTTP.start(uri.host, uri.port,
+                      use_ssl: uri.scheme == 'https') do |http|
+        request = Net::HTTP::Get.new uri
+        http.request request do |response|
+          if (response.code.to_i < 200) || (response.code.to_i > 299)
+            raise StandardError, "Bad CPE def request: #{response.code}: #{response.body}"
+          end
+
+          read_file_chunks(response)
+        end
+      end
+    end
+  end
+
+  def self.read_file_chunks(response)
+    File.open('/data_importer/data/official-cpe-dictionary_v2.2.xml.gz', 'w') do |io|
+      response.read_body do |chunk|
+        io.write chunk.force_encoding('UTF-8')
+      end
+    end
+  end
+
+  def self.transform_node(node)
+    Nokogiri::XML(node.outer_xml).root
+  end
+
+  def self.accept_node(node)
+    node.name == 'cpe-item' && node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+  end
+
+  def self.import(bulk_count = 20000, filepath = '/data_importer/data/official-cpe-dictionary_v2.2.xml.gz')
+    Zlib::GzipReader.open(filepath) do |file|
+      items = []
+      Nokogiri::XML::Reader.from_io(file).each do |node|
+        items << transform_node(node) if accept_node(node)
+
+        if items.count == bulk_count
+          create_cpes(items)
+          items = []
+        end
+      end
+      create_cpes(items) if items.any?
+    rescue Nokogiri::XML::SyntaxError => e
+      if file.nil? == false
+        file.rewind
+        file_content_sample = file.read(400)
+        handle_error("Invalid XML in this file: \"#{file_content_sample}\" - original error #{$ERROR_INFO}")
+      end
+
+      # Couldn't add more info, just re-raise the error
+      raise e
+    end
+  rescue Zlib::GzipFile::Error
+    handle_error("Unable to decompress cpe dictionary: #{$ERROR_INFO}")
+  end
+
+  def self.handle_error(error_message)
+    raise $ERROR_INFO,
+          error_message.to_s,
+          $ERROR_INFO.backtrace
+  end
+
+  def self.create_cpes(items)
+    cpes = items.map do |item|
+      cpe_attrs_from_item(item)
+    end
+
+    Cpe.bulk_insert do |worker|
+      cpes.each do |attrs|
+        worker.add(attrs)
+      end
+    end
+  end
+
+  def self.cpe_attrs_from_item(item)
+    cpe_attrs = {}
+
+    item.search('title').each do |title|
+      cpe_attrs[:title] = title.inner_text if title.attribute('lang').value == 'en-US'
+    end
+
+    metadata = item.at_xpath('meta:item-metadata', XML_NAMESPACES)
+    references = item.search('reference').map { |n| { "#{n.text.gsub(' ', '_').downcase}": n.values } }
+    cpe_attrs[:references] = references
+    cpe_attrs[:name] = item['name'] unless item['name'].nil?
+    cpe_attrs[:modification_date] = metadata['modification-date']
+    cpe_attrs[:status] = metadata['status']
+    cpe_attrs[:nvd_id] = metadata['nvd-id']
+    cpe_attrs
+  end
+
+  def self.create_cpe(item)
+    cpe_attrs = cpe_attrs_from_item(item)
+    cpe = Cpe.where(name: cpe_attrs[:name]).first_or_initialize
+    return unless cpe.new_record?
+
+    cpe.title = cpe_attrs[:title]
+    cpe.metadata = cpe_attrs[:metadata]
+    cpe.references = cpe_attrs[:references]
+    cpe.modification_date = cpe_attrs[:modification_date]
+    cpe.status            = cpe_attrs[:status]
+    cpe.nvd_id            = cpe_attrs[:nvd_id]
+    cpe.save
+  end
+
+  def self.download_and_import
+    download
+    import
+  end
+end

test/controllers/cpes_controller_test.rb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class CpesControllerTest < ActionDispatch::IntegrationTest
+  # test "the truth" do
+  #   assert true
+  # end
+end

test/controllers/cves_controller_test.rb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class CvesControllerTest < ActionDispatch::IntegrationTest
+  # test "the truth" do
+  #   assert true
+  # end
+end