bpmcdevitt/data_importer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adjust cve list importer simplify since we are using upsert_all

Browse source
This commit is contained in:
Brendan McDevitt 2022-04-19 10:46:58 -05:00
parent 2b28810cf2
commit 3d6c30b8a1
2 changed files with 99 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

199
db/schema.rb
View file

@ -1,5 +1,3 @@
# frozen_string_literal: true
# This file is auto-generated from the current state of the database. Instead # This file is auto-generated from the current state of the database. Instead
# of editing this file, please use the migrations feature of Active Record to # of editing this file, please use the migrations feature of Active Record to
# incrementally modify your database, and then regenerate this schema definition. # incrementally modify your database, and then regenerate this schema definition.
@ -12,121 +10,122 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 20_220_411_181_501) do ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension 'plpgsql' enable_extension "plpgsql"
create_table 'cnas', force: :cascade do |t| create_table "cnas", force: :cascade do |t|
t.string 'short_name' t.string "short_name"
t.string 'cna_id' t.string "cna_id"
t.string 'organization_name' t.string "organization_name"
t.string 'scope' t.string "scope"
t.jsonb 'contact' t.jsonb "contact"
t.jsonb 'disclosure_policy' t.jsonb "disclosure_policy"
t.jsonb 'security_advisories' t.jsonb "security_advisories"
t.string 'resources', array: true t.string "resources", array: true
t.jsonb 'cna' t.jsonb "cna"
t.string 'country' t.string "country"
t.datetime 'created_at', null: false t.datetime "created_at", null: false
t.datetime 'updated_at', null: false t.datetime "updated_at", null: false
t.index ['cna_id'], name: 'index_cnas_on_cna_id', unique: true t.index ["cna_id"], name: "index_cnas_on_cna_id", unique: true
end end
create_table 'cpes', force: :cascade do |t| create_table "cpes", force: :cascade do |t|
t.string 'status' t.string "status"
t.date 'modification_date' t.date "modification_date"
t.integer 'nvd_id' t.integer "nvd_id"
t.jsonb 'references' t.jsonb "references"
t.string 'title' t.string "title"
t.string 'name' t.string "name"
t.index ['nvd_id'], name: 'index_cpes_on_nvd_id', unique: true t.index ["nvd_id"], name: "index_cpes_on_nvd_id", unique: true
end end
create_table 'cvemon_cves', force: :cascade do |t| create_table "cvemon_cves", force: :cascade do |t|
t.string 'cve_id' t.string "cve_id"
t.string 'urls', array: true t.string "urls", array: true
t.datetime 'created_at', null: false t.datetime "created_at", null: false
t.datetime 'updated_at', null: false t.datetime "updated_at", null: false
t.index ['cve_id'], name: 'index_cvemon_cves_on_cve_id', unique: true t.index ["cve_id"], name: "index_cvemon_cves_on_cve_id", unique: true
end end
create_table 'cves', force: :cascade do |t| create_table "cves", force: :cascade do |t|
t.jsonb 'cve_data_meta' t.jsonb "cve_data_meta"
t.string 'cve_id' t.string "cve_id"
t.jsonb 'affects' t.jsonb "affects"
t.string 'data_format' t.string "data_format"
t.string 'data_type' t.string "data_type"
t.string 'data_version' t.string "data_version"
t.jsonb 'description' t.jsonb "description"
t.jsonb 'impact' t.jsonb "impact"
t.jsonb 'problemtype' t.jsonb "problemtype"
t.jsonb 'references' t.jsonb "references"
t.jsonb 'source' t.jsonb "source"
t.datetime 'created_at', precision: nil, null: false t.datetime "created_at", precision: nil, null: false
t.datetime 'updated_at', precision: nil, null: false t.datetime "updated_at", precision: nil, null: false
t.index ['cve_id'], name: 'index_cves_on_cve_id', unique: true t.index ["cve_id"], name: "index_cves_on_cve_id", unique: true
end end
create_table 'github_advisories', force: :cascade do |t| create_table "github_advisories", force: :cascade do |t|
t.string 'schema_version' t.string "schema_version"
t.string 'ghsa_id' t.string "ghsa_id"
t.date 'modified' t.date "modified"
t.date 'published' t.date "published"
t.string 'aliases', array: true t.string "aliases", array: true
t.string 'summary' t.string "summary"
t.string 'details' t.string "details"
t.jsonb 'severity' t.jsonb "severity"
t.jsonb 'affected' t.jsonb "affected"
t.jsonb 'references' t.jsonb "references"
t.jsonb 'database_specific' t.jsonb "database_specific"
t.index ['ghsa_id'], name: 'index_github_advisories_on_ghsa_id', unique: true t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true
end end
create_table 'github_pocs', force: :cascade do |t| create_table "github_pocs", force: :cascade do |t|
t.integer 'github_poc_id' t.integer "github_poc_id"
t.string 'cve_id', default: 'None' t.string "cve_id", default: "None"
t.string 'name' t.string "name"
t.string 'full_name' t.string "full_name"
t.jsonb 'owner' t.jsonb "owner"
t.string 'html_url' t.string "html_url"
t.string 'description' t.string "description"
t.boolean 'fork' t.boolean "fork"
t.date 'created_at' t.date "created_at"
t.date 'updated_at' t.date "updated_at"
t.date 'pushed_at' t.date "pushed_at"
t.integer 'stargazers_count' t.integer "stargazers_count"
t.integer 'watchers_count' t.integer "watchers_count"
t.integer 'forks_count' t.integer "forks_count"
t.boolean 'allow_forking' t.boolean "allow_forking"
t.boolean 'is_template' t.boolean "is_template"
t.string 'topics', array: true t.string "topics", array: true
t.string 'visibility' t.string "visibility"
t.integer 'forks' t.integer "forks"
t.integer 'watchers' t.integer "watchers"
t.integer 'score' t.integer "score"
t.index ['github_poc_id'], name: 'index_github_pocs_on_github_poc_id', unique: true t.index ["github_poc_id"], name: "index_github_pocs_on_github_poc_id", unique: true
end end
create_table 'github_users', force: :cascade do |t| create_table "github_users", force: :cascade do |t|
t.string 'github_id' t.string "github_id"
t.string 'login' t.string "login"
t.string 'name' t.string "name"
t.string 'avatar_url' t.string "avatar_url"
t.string 'bio' t.string "bio"
t.text 'bio_html' t.text "bio_html"
t.string 'location' t.string "location"
t.jsonb 'repositories' t.jsonb "repositories"
end end
create_table 'inthewild_cve_exploits', force: :cascade do |t| create_table "inthewild_cve_exploits", force: :cascade do |t|
t.string 'cve_id' t.string "cve_id"
t.string 'earliest_report' t.string "earliest_report"
end end
create_table 'trickest_poc_cves', force: :cascade do |t| create_table "trickest_poc_cves", force: :cascade do |t|
t.string 'cve_id' t.string "cve_id"
t.string 'cve_url' t.string "cve_url"
t.string 'description' t.string "description"
t.string 'poc_links', array: true t.string "poc_links", array: true
end end
end end

7
lib/importers/cve_list_importer.rb
View file

@ -54,14 +54,7 @@ class CveListImporter < GithubRepo
puts '----------' * 12 puts '----------' * 12
(1999..Date.today.year).map do |year| (1999..Date.today.year).map do |year|
cves = read_jsons_for_year(year) cves = read_jsons_for_year(year)
# ids = cves.map { |cve| cve[:cve_id] }
# cve_ids_in_db = Cve.where(:cve_id => ids).pluck(:cve_id)
# new_cve_ids = ids - cve_ids_in_db
# new_cves = cves.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) }
puts "Importing any new CVEs from #{year}" puts "Importing any new CVEs from #{year}"
Cve.upsert_all(cves, unique_by: :cve_id) Cve.upsert_all(cves, unique_by: :cve_id)
end end
end end