From 2b28810cf2595a39f974084affbd9c571faa47fd Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Tue, 19 Apr 2022 02:37:27 -0500 Subject: [PATCH] rubocop commit --- Gemfile | 29 +-- Gemfile.lock | 3 + Rakefile | 2 + app/channels/application_cable/channel.rb | 2 + app/channels/application_cable/connection.rb | 2 + app/controllers/application_controller.rb | 2 + app/controllers/cnas_controller.rb | 2 + app/controllers/cpes_controller.rb | 2 + app/controllers/cvemon_cves_controller.rb | 7 +- app/controllers/cves_controller.rb | 2 + .../github_advisories_controller.rb | 2 + app/controllers/github_pocs_controller.rb | 7 +- app/controllers/github_users_controller.rb | 2 + .../inthewild_cve_exploits_controller.rb | 4 +- .../trickest_poc_cves_controller.rb | 7 +- app/helpers/application_helper.rb | 2 + app/helpers/cpes_helper.rb | 2 + app/helpers/cves_helper.rb | 2 + app/jobs/application_job.rb | 2 + app/mailers/application_mailer.rb | 2 + app/models/application_record.rb | 2 + app/models/cna.rb | 4 +- app/models/cpe.rb | 4 +- app/models/cve.rb | 30 +-- app/models/cvemon_cve.rb | 8 +- app/models/github_advisory.rb | 14 +- app/models/github_poc.rb | 8 +- app/models/github_user.rb | 4 +- app/models/inthewild_cve_exploit.rb | 2 + app/models/trickest_poc_cve.rb | 8 +- app/workers/cna_importer_worker.rb | 6 +- app/workers/cpe_importer_worker.rb | 6 +- app/workers/cve_list_importer_worker.rb | 6 +- app/workers/cvemon_cve_importer_worker.rb | 6 +- .../github_advisory_importer_worker.rb | 6 +- app/workers/github_user_importer_worker.rb | 6 +- .../inthewild_cve_exploit_importer_worker.rb | 6 +- app/workers/poc_in_github_importer_worker.rb | 6 +- .../trickest_poc_cve_importer_worker.rb | 6 +- bin/bundle | 2 + bin/rails | 2 + bin/rake | 2 + bin/setup | 2 + bin/update | 2 + bin/yarn | 14 +- config.ru | 2 + config/application.rb | 2 + config/boot.rb | 2 + config/environment.rb | 2 + config/environments/development.rb | 2 + config/environments/production.rb | 8 +- config/environments/test.rb | 2 + .../application_controller_renderer.rb | 1 + config/initializers/assets.rb | 2 + config/initializers/backtrace_silencers.rb | 1 + .../initializers/content_security_policy.rb | 1 + config/initializers/cookies_serializer.rb | 2 + .../initializers/filter_parameter_logging.rb | 2 + config/initializers/inflections.rb | 1 + config/initializers/mime_types.rb | 1 + config/initializers/twitter_config.rb | 10 +- config/initializers/wrap_parameters.rb | 2 + config/puma.rb | 8 +- config/routes.rb | 57 ++--- config/spring.rb | 2 + db/migrate/20220401173431_create_cves.rb | 30 +-- db/migrate/20220404150811_create_cpes.rb | 2 + db/migrate/20220405230622_github_pocs.rb | 4 +- .../20220406064613_inthewild_cve_exploits.rb | 4 +- ...20220407051821_create_trickest_poc_cves.rb | 2 + .../20220407083218_create_cvemon_cves.rb | 2 + db/migrate/20220407223152_create_cnas.rb | 2 + .../20220411174826_create_github_users.rb | 2 + ...20220411181501_create_github_advisories.rb | 2 + db/schema.rb | 199 ++++++++-------- db/seeds.rb | 76 +++--- lib/github_api/github_api.rb | 14 +- lib/github_api/owner_repos.rb | 36 +-- lib/github_api/security_advisory.rb | 26 ++- lib/github_api/user.rb | 26 ++- lib/importers/cna_importer.rb | 35 +-- lib/importers/cpe_importer.rb | 4 +- lib/importers/cve_list_importer.rb | 52 +++-- lib/importers/cvemon_cve_importer.rb | 53 +++-- lib/importers/github_advisory_importer.rb | 136 +++++------ lib/importers/github_repo.rb | 60 ++--- lib/importers/github_user_importer.rb | 15 +- .../inthewild_cve_exploit_importer.rb | 85 +++---- lib/importers/poc_in_github_importer.rb | 27 +-- lib/importers/trickest_poc_cve_importer.rb | 220 +++++++++--------- lib/json_helper.rb | 8 +- lib/tasks/github_usernames_populate.rb | 6 +- test/application_system_test_case.rb | 4 +- test/controllers/cpes_controller_test.rb | 4 +- test/controllers/cves_controller_test.rb | 4 +- test/test_helper.rb | 12 +- 96 files changed, 845 insertions(+), 664 deletions(-) diff --git a/Gemfile b/Gemfile index c1b0d91..9798107 100644 --- a/Gemfile +++ b/Gemfile @@ -1,28 +1,31 @@ +# frozen_string_literal: true + source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby '2.7.0' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 7.0.0' -gem 'actionpack' -gem 'sass-rails' -gem 'railties' -gem 'rest-client' -gem 'twitter' -gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support -gem 'nokogiri' +gem 'actionpack' +gem 'faktory_worker_ruby' gem 'graphql' gem 'graphql-client' +gem 'nokogiri' +gem 'rails', '~> 7.0.0' +gem 'railties' +gem 'rest-client' gem 'retryable' gem 'rubocop' +gem 'rubocop-graphql' gem 'rubocop-rails' -gem 'faktory_worker_ruby' +gem 'sass-rails' +gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support +gem 'twitter' # Use postgres as the database for Active Record -gem 'pg' gem 'bulk_insert' gem 'git' +gem 'pg' # Use Puma as the app server gem 'puma', '~> 3.11' # Use Uglifier as compressor for JavaScript assets @@ -52,8 +55,8 @@ gem 'bootsnap', '>= 1.1.0', require: false group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console - gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] gem 'awesome_print' # pretty print ruby objects + gem 'byebug', platforms: %i[mri mingw x64_mingw] gem 'pry' # Console with powerful introspection capabilities gem 'pry-byebug' # Integrates pry with byebug gem 'pry-doc' # Provide MRI Core documentation @@ -64,8 +67,8 @@ end group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. - gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + gem 'web-console', '>= 3.3.0' # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' gem 'spring-watcher-listen', '~> 2.0.0' @@ -80,4 +83,4 @@ group :test do end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem -gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] diff --git a/Gemfile.lock b/Gemfile.lock index d9c3d6f..b05d148 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -296,6 +296,8 @@ GEM unicode-display_width (>= 1.4.0, < 3.0) rubocop-ast (1.17.0) parser (>= 3.1.1.0) + rubocop-graphql (0.14.2) + rubocop (>= 0.87, < 2) rubocop-rails (2.14.2) activesupport (>= 4.2.0) rack (>= 1.1) @@ -404,6 +406,7 @@ DEPENDENCIES rest-client retryable rubocop + rubocop-graphql rubocop-rails sass-rails selenium-webdriver diff --git a/Rakefile b/Rakefile index e85f913..488c551 100644 --- a/Rakefile +++ b/Rakefile @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Add your own tasks in files placed in lib/tasks ending in .rake, # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. diff --git a/app/channels/application_cable/channel.rb b/app/channels/application_cable/channel.rb index d672697..9aec230 100644 --- a/app/channels/application_cable/channel.rb +++ b/app/channels/application_cable/channel.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ApplicationCable class Channel < ActionCable::Channel::Base end diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb index 0ff5442..8d6c2a1 100644 --- a/app/channels/application_cable/connection.rb +++ b/app/channels/application_cable/connection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ApplicationCable class Connection < ActionCable::Connection::Base end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 09705d1..7944f9f 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class ApplicationController < ActionController::Base end diff --git a/app/controllers/cnas_controller.rb b/app/controllers/cnas_controller.rb index 9da836c..14fe9db 100644 --- a/app/controllers/cnas_controller.rb +++ b/app/controllers/cnas_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CnasController < ApplicationController def index @cnas = Cna.all diff --git a/app/controllers/cpes_controller.rb b/app/controllers/cpes_controller.rb index b2cff81..ef744b9 100644 --- a/app/controllers/cpes_controller.rb +++ b/app/controllers/cpes_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CpesController < ApplicationController def index @cpes = Cpe.all diff --git a/app/controllers/cvemon_cves_controller.rb b/app/controllers/cvemon_cves_controller.rb index 8205191..8872e13 100644 --- a/app/controllers/cvemon_cves_controller.rb +++ b/app/controllers/cvemon_cves_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CvemonCvesController < ApplicationController def index @pocs = CvemonCve.all @@ -5,12 +7,12 @@ class CvemonCvesController < ApplicationController end def show - @poc = CvemonCve.find_by(:id => params[:id]) + @poc = CvemonCve.find_by(id: params[:id]) render json: @poc.to_json end def show_for_cve - @poc = CvemonCve.where(:cve_id => params[:cve_id]) + @poc = CvemonCve.where(cve_id: params[:cve_id]) render json: @poc.to_json end @@ -18,5 +20,4 @@ class CvemonCvesController < ApplicationController @cves_for_year = CvemonCve.from_year(params[:year]) render json: @cves_for_year.to_json end - end diff --git a/app/controllers/cves_controller.rb b/app/controllers/cves_controller.rb index 0ad98e7..ff72411 100644 --- a/app/controllers/cves_controller.rb +++ b/app/controllers/cves_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CvesController < ApplicationController def index @cves = Cve.all diff --git a/app/controllers/github_advisories_controller.rb b/app/controllers/github_advisories_controller.rb index 7a7ab01..40c3c13 100644 --- a/app/controllers/github_advisories_controller.rb +++ b/app/controllers/github_advisories_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class GithubAdvisoriesController < ApplicationController def index @advisories = GithubAdvisory.all diff --git a/app/controllers/github_pocs_controller.rb b/app/controllers/github_pocs_controller.rb index 75c2ca1..5b5d007 100644 --- a/app/controllers/github_pocs_controller.rb +++ b/app/controllers/github_pocs_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class GithubPocsController < ApplicationController def index @pocs = GithubPoc.all @@ -5,12 +7,12 @@ class GithubPocsController < ApplicationController end def show - @poc = GithubPoc.find_by(:id => params[:id]) + @poc = GithubPoc.find_by(id: params[:id]) render json: @poc.to_json end def show_for_cve - @poc = GithubPoc.where(:cve_id => params[:cve_id]) + @poc = GithubPoc.where(cve_id: params[:cve_id]) render json: @poc.to_json end @@ -18,5 +20,4 @@ class GithubPocsController < ApplicationController @cves_for_year = GithubPoc.from_year(params[:year]) render json: @cves_for_year.to_json end - end diff --git a/app/controllers/github_users_controller.rb b/app/controllers/github_users_controller.rb index 8dc85ba..6c1ba10 100644 --- a/app/controllers/github_users_controller.rb +++ b/app/controllers/github_users_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class GithubUsersController < ApplicationController def index @users = GithubUser.all diff --git a/app/controllers/inthewild_cve_exploits_controller.rb b/app/controllers/inthewild_cve_exploits_controller.rb index c4ac4a3..7dd37c2 100644 --- a/app/controllers/inthewild_cve_exploits_controller.rb +++ b/app/controllers/inthewild_cve_exploits_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class InthewildCveExploitsController < ApplicationController def index @cves = InthewildCveExploit.all @@ -5,7 +7,7 @@ class InthewildCveExploitsController < ApplicationController end def show - @cve = InthewildCveExploit.find_by(:cve_id => params[:cve_id]) + @cve = InthewildCveExploit.find_by(cve_id: params[:cve_id]) render json: @cve.to_json end end diff --git a/app/controllers/trickest_poc_cves_controller.rb b/app/controllers/trickest_poc_cves_controller.rb index d35ef46..dde2b7a 100644 --- a/app/controllers/trickest_poc_cves_controller.rb +++ b/app/controllers/trickest_poc_cves_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class TrickestPocCvesController < ApplicationController def index @pocs = TrickestPocCve.all @@ -5,12 +7,12 @@ class TrickestPocCvesController < ApplicationController end def show - @poc = TrickestPocCve.find_by(:id => params[:id]) + @poc = TrickestPocCve.find_by(id: params[:id]) render json: @poc.to_json end def show_for_cve - @poc = TrickestPocCve.where(:cve_id => params[:cve_id]) + @poc = TrickestPocCve.where(cve_id: params[:cve_id]) render json: @poc.to_json end @@ -18,5 +20,4 @@ class TrickestPocCvesController < ApplicationController @cves_for_year = TrickestPocCve.from_year(params[:year]) render json: @cves_for_year.to_json end - end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index de6be79..15b06f0 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + module ApplicationHelper end diff --git a/app/helpers/cpes_helper.rb b/app/helpers/cpes_helper.rb index b9a5e55..9240b50 100644 --- a/app/helpers/cpes_helper.rb +++ b/app/helpers/cpes_helper.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + module CpesHelper end diff --git a/app/helpers/cves_helper.rb b/app/helpers/cves_helper.rb index c1f2986..2432a41 100644 --- a/app/helpers/cves_helper.rb +++ b/app/helpers/cves_helper.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + module CvesHelper end diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index a009ace..d92ffdd 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class ApplicationJob < ActiveJob::Base end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index 286b223..d84cb6e 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ApplicationMailer < ActionMailer::Base default from: 'from@example.com' layout 'mailer' diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 10a4cba..71fbba5 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ApplicationRecord < ActiveRecord::Base self.abstract_class = true end diff --git a/app/models/cna.rb b/app/models/cna.rb index 1836e61..600b13e 100644 --- a/app/models/cna.rb +++ b/app/models/cna.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + class Cna < ActiveRecord::Base def self.find_by_cna_id(cna_id) - find_by(:cna_id => cna_id) + find_by(cna_id: cna_id) end end diff --git a/app/models/cpe.rb b/app/models/cpe.rb index f348b8e..73373eb 100644 --- a/app/models/cpe.rb +++ b/app/models/cpe.rb @@ -1 +1,3 @@ -class Cpe < ActiveRecord::Base; end \ No newline at end of file +# frozen_string_literal: true + +class Cpe < ActiveRecord::Base; end diff --git a/app/models/cve.rb b/app/models/cve.rb index b6336f8..cbd340b 100644 --- a/app/models/cve.rb +++ b/app/models/cve.rb @@ -1,16 +1,18 @@ +# frozen_string_literal: true + class Cve < ActiveRecord::Base - scope :with_reserved, -> { where("cve_data_meta->>'STATE' = 'RESERVED'") } - scope :without_reserved, -> { where.not("cve_data_meta->>'STATE' = 'RESERVED'") } - scope :with_rejected, -> { where("cve_data_meta->>'STATE' = 'REJECT'") } - scope :without_rejected, -> { where.not("cve_data_meta->>'STATE' = 'REJECT'") } - scope :with_public, -> { where("cve_data_meta->>'STATE' = 'PUBLIC'") } - scope :without_public, -> { where.not("cve_data_meta->>'STATE' = 'PUBLIC'") } - - def self.find_by_id(id) - find_by(:cve_id => id) - end - - def self.from_year(year) - where("cve_id LIKE ?", "CVE-#{year}-%") - end + scope :with_reserved, -> { where("cve_data_meta->>'STATE' = 'RESERVED'") } + scope :without_reserved, -> { where.not("cve_data_meta->>'STATE' = 'RESERVED'") } + scope :with_rejected, -> { where("cve_data_meta->>'STATE' = 'REJECT'") } + scope :without_rejected, -> { where.not("cve_data_meta->>'STATE' = 'REJECT'") } + scope :with_public, -> { where("cve_data_meta->>'STATE' = 'PUBLIC'") } + scope :without_public, -> { where.not("cve_data_meta->>'STATE' = 'PUBLIC'") } + + def self.find_by_id(id) + find_by(cve_id: id) + end + + def self.from_year(year) + where('cve_id LIKE ?', "CVE-#{year}-%") + end end diff --git a/app/models/cvemon_cve.rb b/app/models/cvemon_cve.rb index d7a4931..575c4ca 100644 --- a/app/models/cvemon_cve.rb +++ b/app/models/cvemon_cve.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + class CvemonCve < ActiveRecord::Base - def self.from_year(year) - where("cve_id LIKE ?", "CVE-#{year}-%") - end + def self.from_year(year) + where('cve_id LIKE ?', "CVE-#{year}-%") + end end diff --git a/app/models/github_advisory.rb b/app/models/github_advisory.rb index 6f5e14b..81e11a6 100644 --- a/app/models/github_advisory.rb +++ b/app/models/github_advisory.rb @@ -1,8 +1,10 @@ -class GithubAdvisory< ActiveRecord::Base - scope :github_reviewed, -> { where("database_specific->>'github_reviewed' = 'true'") } - scope :unreviewed, -> { where("database_specific->>'github_reviewed' = 'false'") } +# frozen_string_literal: true - def self.find_by_ghsa_id(ghsa_id) - find_by(ghsa_id: ghsa_id) - end +class GithubAdvisory < ActiveRecord::Base + scope :github_reviewed, -> { where("database_specific->>'github_reviewed' = 'true'") } + scope :unreviewed, -> { where("database_specific->>'github_reviewed' = 'false'") } + + def self.find_by_ghsa_id(ghsa_id) + find_by(ghsa_id: ghsa_id) + end end diff --git a/app/models/github_poc.rb b/app/models/github_poc.rb index f804d89..1905ec7 100644 --- a/app/models/github_poc.rb +++ b/app/models/github_poc.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + class GithubPoc < ActiveRecord::Base - def self.from_year(year) - where("cve_id LIKE ?", "CVE-#{year}-%") - end + def self.from_year(year) + where('cve_id LIKE ?', "CVE-#{year}-%") + end end diff --git a/app/models/github_user.rb b/app/models/github_user.rb index ed2de26..db9266e 100644 --- a/app/models/github_user.rb +++ b/app/models/github_user.rb @@ -1,4 +1,6 @@ -class GithubUser< ActiveRecord::Base +# frozen_string_literal: true + +class GithubUser < ActiveRecord::Base def self.find_by_username(username) find_by(login: username) end diff --git a/app/models/inthewild_cve_exploit.rb b/app/models/inthewild_cve_exploit.rb index 6f060bd..df335e2 100644 --- a/app/models/inthewild_cve_exploit.rb +++ b/app/models/inthewild_cve_exploit.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class InthewildCveExploit < ActiveRecord::Base end diff --git a/app/models/trickest_poc_cve.rb b/app/models/trickest_poc_cve.rb index 4aa7d69..d5a9989 100644 --- a/app/models/trickest_poc_cve.rb +++ b/app/models/trickest_poc_cve.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + class TrickestPocCve < ActiveRecord::Base - def self.from_year(year) - where("cve_id LIKE ?", "CVE-#{year}-%") - end + def self.from_year(year) + where('cve_id LIKE ?', "CVE-#{year}-%") + end end diff --git a/app/workers/cna_importer_worker.rb b/app/workers/cna_importer_worker.rb index c32b3ab..66e1d30 100644 --- a/app/workers/cna_importer_worker.rb +++ b/app/workers/cna_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/cna_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/cna_importer' class CnaImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class CnaImporterWorker puts "Hello, I am #{jid} with args #{args}" CnaImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/cpe_importer_worker.rb b/app/workers/cpe_importer_worker.rb index 6d67c74..8503e44 100644 --- a/app/workers/cpe_importer_worker.rb +++ b/app/workers/cpe_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/cpe_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/cpe_importer' class CpeImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class CpeImporterWorker puts "Hello, I am #{jid} with args #{args}" CpeImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/cve_list_importer_worker.rb b/app/workers/cve_list_importer_worker.rb index b3cdb21..19693b5 100644 --- a/app/workers/cve_list_importer_worker.rb +++ b/app/workers/cve_list_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/cve_list_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/cve_list_importer' class CveListImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class CveListImporterWorker puts "Hello, I am #{jid} with args #{args}" CveListImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/cvemon_cve_importer_worker.rb b/app/workers/cvemon_cve_importer_worker.rb index 4e1e90e..088aeab 100644 --- a/app/workers/cvemon_cve_importer_worker.rb +++ b/app/workers/cvemon_cve_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/cvemon_cve_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/cvemon_cve_importer' class CvemonCveImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class CvemonCveImporterWorker puts "Hello, I am #{jid} with args #{args}" CvemonCveImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/github_advisory_importer_worker.rb b/app/workers/github_advisory_importer_worker.rb index 62f818a..c95c8a0 100644 --- a/app/workers/github_advisory_importer_worker.rb +++ b/app/workers/github_advisory_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/github_advisory_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/github_advisory_importer' class GithubAdvisoryImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class GithubAdvisoryImporterWorker puts "Hello, I am #{jid} with args #{args}" GithubAdvisoryImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/github_user_importer_worker.rb b/app/workers/github_user_importer_worker.rb index dae9faf..293a785 100644 --- a/app/workers/github_user_importer_worker.rb +++ b/app/workers/github_user_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/github_user_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/github_user_importer' class GithubUserImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class GithubUserImporterWorker puts "Hello, I am #{jid} with args #{args}" GithubUserImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/inthewild_cve_exploit_importer_worker.rb b/app/workers/inthewild_cve_exploit_importer_worker.rb index 86535e0..1fe7501 100644 --- a/app/workers/inthewild_cve_exploit_importer_worker.rb +++ b/app/workers/inthewild_cve_exploit_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/inthewild_cve_exploit_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/inthewild_cve_exploit_importer' class InthewildCveExploitImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class InthewildCveExploitImporterWorker puts "Hello, I am #{jid} with args #{args}" InthewildCveExploitImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/poc_in_github_importer_worker.rb b/app/workers/poc_in_github_importer_worker.rb index 3144e0e..8936ce6 100644 --- a/app/workers/poc_in_github_importer_worker.rb +++ b/app/workers/poc_in_github_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/poc_in_github_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/poc_in_github_importer' class PocInGithubImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class PocInGithubImporterWorker puts "Hello, I am #{jid} with args #{args}" PocInGithubImporter.new.import end -end \ No newline at end of file +end diff --git a/app/workers/trickest_poc_cve_importer_worker.rb b/app/workers/trickest_poc_cve_importer_worker.rb index 0a4bd9c..74c192a 100644 --- a/app/workers/trickest_poc_cve_importer_worker.rb +++ b/app/workers/trickest_poc_cve_importer_worker.rb @@ -1,4 +1,6 @@ -require '/data_importer/lib/importers/trickest_poc_cve_importer.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/trickest_poc_cve_importer' class TrickestPocCveImporterWorker include Faktory::Job @@ -7,4 +9,4 @@ class TrickestPocCveImporterWorker puts "Hello, I am #{jid} with args #{args}" TrickestPocCveImporter.new.import end -end \ No newline at end of file +end diff --git a/bin/bundle b/bin/bundle index f19acf5..2dbb717 100755 --- a/bin/bundle +++ b/bin/bundle @@ -1,3 +1,5 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) load Gem.bin_path('bundler', 'bundle') diff --git a/bin/rails b/bin/rails index 0739660..a31728a 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,6 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + APP_PATH = File.expand_path('../config/application', __dir__) require_relative '../config/boot' require 'rails/commands' diff --git a/bin/rake b/bin/rake index 1724048..c199955 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,6 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/setup b/bin/setup index 94fd4d7..c2e43ce 100755 --- a/bin/setup +++ b/bin/setup @@ -1,4 +1,6 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + require 'fileutils' include FileUtils diff --git a/bin/update b/bin/update index 58bfaed..313c74b 100755 --- a/bin/update +++ b/bin/update @@ -1,4 +1,6 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + require 'fileutils' include FileUtils diff --git a/bin/yarn b/bin/yarn index 460dd56..4cac416 100755 --- a/bin/yarn +++ b/bin/yarn @@ -1,11 +1,11 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + APP_ROOT = File.expand_path('..', __dir__) Dir.chdir(APP_ROOT) do - begin - exec "yarnpkg", *ARGV - rescue Errno::ENOENT - $stderr.puts "Yarn executable was not detected in the system." - $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" - exit 1 - end + exec 'yarnpkg', *ARGV +rescue Errno::ENOENT + warn 'Yarn executable was not detected in the system.' + warn 'Download Yarn at https://yarnpkg.com/en/docs/install' + exit 1 end diff --git a/config.ru b/config.ru index f7ba0b5..842bccc 100644 --- a/config.ru +++ b/config.ru @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This file is used by Rack-based servers to start the application. require_relative 'config/environment' diff --git a/config/application.rb b/config/application.rb index bafde52..c0b69f3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative 'boot' require 'rails/all' diff --git a/config/boot.rb b/config/boot.rb index b9e460c..c04863f 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) require 'bundler/setup' # Set up gems listed in the Gemfile. diff --git a/config/environment.rb b/config/environment.rb index 426333b..d5abe55 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Load the Rails application. require_relative 'application' diff --git a/config/environments/development.rb b/config/environments/development.rb index 1311e3e..33c8acf 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. diff --git a/config/environments/production.rb b/config/environments/production.rb index 2ab83a7..cb0ad29 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -54,7 +56,7 @@ Rails.application.configure do config.log_level = :debug # Prepend all log lines with the following tags. - config.log_tags = [ :request_id ] + config.log_tags = [:request_id] # Use a different cache store in production. # config.cache_store = :mem_cache_store @@ -83,8 +85,8 @@ Rails.application.configure do # require 'syslog/logger' # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') - if ENV["RAILS_LOG_TO_STDOUT"].present? - logger = ActiveSupport::Logger.new(STDOUT) + if ENV['RAILS_LOG_TO_STDOUT'].present? + logger = ActiveSupport::Logger.new($stdout) logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end diff --git a/config/environments/test.rb b/config/environments/test.rb index 0a38fd3..3091ac4 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. diff --git a/config/initializers/application_controller_renderer.rb b/config/initializers/application_controller_renderer.rb index 89d2efa..f4556db 100644 --- a/config/initializers/application_controller_renderer.rb +++ b/config/initializers/application_controller_renderer.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # Be sure to restart your server when you modify this file. # ActiveSupport::Reloader.to_prepare do diff --git a/config/initializers/assets.rb b/config/initializers/assets.rb index 4b828e8..a9b0d0f 100644 --- a/config/initializers/assets.rb +++ b/config/initializers/assets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Be sure to restart your server when you modify this file. # Version of your assets, change this if you want to expire all your assets. diff --git a/config/initializers/backtrace_silencers.rb b/config/initializers/backtrace_silencers.rb index 59385cd..d0f0d3b 100644 --- a/config/initializers/backtrace_silencers.rb +++ b/config/initializers/backtrace_silencers.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index d3bcaa5..497f566 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # Be sure to restart your server when you modify this file. # Define an application-wide content security policy diff --git a/config/initializers/cookies_serializer.rb b/config/initializers/cookies_serializer.rb index 5a6a32d..ee8dff9 100644 --- a/config/initializers/cookies_serializer.rb +++ b/config/initializers/cookies_serializer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Be sure to restart your server when you modify this file. # Specify a serializer for the signed and encrypted cookie jars. diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 4a994e1..7a4f47b 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Be sure to restart your server when you modify this file. # Configure sensitive parameters which will be filtered from the log file. diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ac033bf..aa7435f 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # Be sure to restart your server when you modify this file. # Add new inflection rules using the following format. Inflections diff --git a/config/initializers/mime_types.rb b/config/initializers/mime_types.rb index dc18996..6e1d16f 100644 --- a/config/initializers/mime_types.rb +++ b/config/initializers/mime_types.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # Be sure to restart your server when you modify this file. # Add new mime types for use in respond_to blocks: diff --git a/config/initializers/twitter_config.rb b/config/initializers/twitter_config.rb index 456eeb7..0eb2a62 100644 --- a/config/initializers/twitter_config.rb +++ b/config/initializers/twitter_config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'twitter' require 'tweetkit' @@ -9,7 +11,7 @@ twitter_client = Twitter::REST::Client.new do |config| end tweetkit_client = Tweetkit::Client.new do |config| - config.bearer_token = ENV['twitter_bearer_token'] - config.consumer_key = ENV['twitter_api_key'] - config.consumer_secret = ENV['twitter_api_key_secret'] -end \ No newline at end of file + config.bearer_token = ENV['twitter_bearer_token'] + config.consumer_key = ENV['twitter_api_key'] + config.consumer_secret = ENV['twitter_api_key_secret'] +end diff --git a/config/initializers/wrap_parameters.rb b/config/initializers/wrap_parameters.rb index bbfc396..2f3c0db 100644 --- a/config/initializers/wrap_parameters.rb +++ b/config/initializers/wrap_parameters.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Be sure to restart your server when you modify this file. # This file contains settings for ActionController::ParamsWrapper which diff --git a/config/puma.rb b/config/puma.rb index a5eccf8..d9a94f3 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,19 +1,21 @@ +# frozen_string_literal: true + # Puma can serve each request in a thread from an internal thread pool. # The `threads` method setting takes two numbers: a minimum and maximum. # Any libraries that use thread pools should be configured to match # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. # -threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } +threads_count = ENV.fetch('RAILS_MAX_THREADS', 5) threads threads_count, threads_count # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # -port ENV.fetch("PORT") { 3000 } +port ENV.fetch('PORT', 3000) # Specifies the `environment` that Puma will run in. # -environment ENV.fetch("RAILS_ENV") { "development" } +environment ENV.fetch('RAILS_ENV', 'development') # Specifies the number of `workers` to boot in clustered mode. # Workers are forked webserver processes. If using threads and workers together diff --git a/config/routes.rb b/config/routes.rb index 6dc453d..7370b4a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,39 +1,40 @@ +# frozen_string_literal: true + Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html - get "/cves", to: "cves#index" - get "/cves/:cve_id", to: "cves#show" - get "/cves/years/:year", to: "cves#show_year" + get '/cves', to: 'cves#index' + get '/cves/:cve_id', to: 'cves#show' + get '/cves/years/:year', to: 'cves#show_year' - get "/cpes", to: "cpes#index" - get "/cpes/:id", to: "cpes#show" + get '/cpes', to: 'cpes#index' + get '/cpes/:id', to: 'cpes#show' - get "/github_pocs", to: "github_pocs#index" - get "/github_pocs/:id", to: "github_pocs#show" - get "/github_pocs/cve/:cve_id", to: "github_pocs#show_for_cve" - get "/github_pocs/years/:year", to: "github_pocs#show_year" + get '/github_pocs', to: 'github_pocs#index' + get '/github_pocs/:id', to: 'github_pocs#show' + get '/github_pocs/cve/:cve_id', to: 'github_pocs#show_for_cve' + get '/github_pocs/years/:year', to: 'github_pocs#show_year' - get "/inthewild_cve_exploits", to: "inthewild_cve_exploits#index" - get "/inthewild_cve_exploits/:cve_id", to: "inthewild_cve_exploits#show" + get '/inthewild_cve_exploits', to: 'inthewild_cve_exploits#index' + get '/inthewild_cve_exploits/:cve_id', to: 'inthewild_cve_exploits#show' - get "/trickest_poc_cves", to: "trickest_poc_cves#index" - get "/trickest_poc_cves/:id", to: "trickest_poc_cves#show" - get "/trickest_poc_cves/cve/:cve_id", to: "trickest_poc_cves#show_for_cve" - get "/trickest_poc_cves/years/:year", to: "trickest_poc_cves#show_year" + get '/trickest_poc_cves', to: 'trickest_poc_cves#index' + get '/trickest_poc_cves/:id', to: 'trickest_poc_cves#show' + get '/trickest_poc_cves/cve/:cve_id', to: 'trickest_poc_cves#show_for_cve' + get '/trickest_poc_cves/years/:year', to: 'trickest_poc_cves#show_year' - get "/cvemon_cves", to: "cvemon_cves#index" - get "/cvemon_cves/:id", to: "cvemon_cves#show" - get "/cvemon_cves/cve/:cve_id", to: "cvemon_cves#show_for_cve" - get "/cvemon_cves/years/:year", to: "cvemon_cves#show_year" + get '/cvemon_cves', to: 'cvemon_cves#index' + get '/cvemon_cves/:id', to: 'cvemon_cves#show' + get '/cvemon_cves/cve/:cve_id', to: 'cvemon_cves#show_for_cve' + get '/cvemon_cves/years/:year', to: 'cvemon_cves#show_year' - get "/cnas", to: "cnas#index" - get "/cnas/:id", to: "cnas#show" - get "/cnas/cna/:cna_id", to: "cnas#show_for_cna" - get "/cnas/organization_name/:organization_name", to: "cnas#show_for_orgname" + get '/cnas', to: 'cnas#index' + get '/cnas/:id', to: 'cnas#show' + get '/cnas/cna/:cna_id', to: 'cnas#show_for_cna' + get '/cnas/organization_name/:organization_name', to: 'cnas#show_for_orgname' - get "/github_advisories", to: "github_advisories#index" - get "/github_advisories/:ghsa_id", to: "github_advisories#show" - - get "/github_users", to: "github_users#index" - get "/github_users/:username", to: "github_users#show" + get '/github_advisories', to: 'github_advisories#index' + get '/github_advisories/:ghsa_id', to: 'github_advisories#show' + get '/github_users', to: 'github_users#index' + get '/github_users/:username', to: 'github_users#show' end diff --git a/config/spring.rb b/config/spring.rb index 9fa7863..c5933e4 100644 --- a/config/spring.rb +++ b/config/spring.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + %w[ .ruby-version .rbenv-vars diff --git a/db/migrate/20220401173431_create_cves.rb b/db/migrate/20220401173431_create_cves.rb index c4e4ad9..4699bae 100644 --- a/db/migrate/20220401173431_create_cves.rb +++ b/db/migrate/20220401173431_create_cves.rb @@ -1,19 +1,21 @@ +# frozen_string_literal: true + class CreateCves < ActiveRecord::Migration[5.2] def change create_table :cves do |t| - t.jsonb :cve_data_meta - t.string :cve_id - t.index :cve_id, unique: true - t.jsonb :affects - t.string :data_format - t.string :data_type - t.string :data_version - t.jsonb :description - t.jsonb :impact - t.jsonb :problemtype - t.jsonb :references - t.jsonb :source - t.timestamps - end + t.jsonb :cve_data_meta + t.string :cve_id + t.index :cve_id, unique: true + t.jsonb :affects + t.string :data_format + t.string :data_type + t.string :data_version + t.jsonb :description + t.jsonb :impact + t.jsonb :problemtype + t.jsonb :references + t.jsonb :source + t.timestamps + end end end diff --git a/db/migrate/20220404150811_create_cpes.rb b/db/migrate/20220404150811_create_cpes.rb index 92b87e3..ef2c72e 100644 --- a/db/migrate/20220404150811_create_cpes.rb +++ b/db/migrate/20220404150811_create_cpes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateCpes < ActiveRecord::Migration[7.0] def change create_table :cpes do |t| diff --git a/db/migrate/20220405230622_github_pocs.rb b/db/migrate/20220405230622_github_pocs.rb index b191395..3420dc2 100644 --- a/db/migrate/20220405230622_github_pocs.rb +++ b/db/migrate/20220405230622_github_pocs.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + class GithubPocs < ActiveRecord::Migration[7.0] def change create_table :github_pocs do |t| t.integer :github_poc_id t.index :github_poc_id, unique: true - t.string :cve_id, default: "None" + t.string :cve_id, default: 'None' t.string :name t.string :full_name t.jsonb :owner diff --git a/db/migrate/20220406064613_inthewild_cve_exploits.rb b/db/migrate/20220406064613_inthewild_cve_exploits.rb index 2bcf49a..38fc846 100644 --- a/db/migrate/20220406064613_inthewild_cve_exploits.rb +++ b/db/migrate/20220406064613_inthewild_cve_exploits.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + class InthewildCveExploits < ActiveRecord::Migration[7.0] def change create_table :inthewild_cve_exploits do |t| t.string :cve_id # i think maybe making a string is better for now for the earliestReport data t.string :earliest_report - #t.date :earliest_report + # t.date :earliest_report end end end diff --git a/db/migrate/20220407051821_create_trickest_poc_cves.rb b/db/migrate/20220407051821_create_trickest_poc_cves.rb index 8fda9fd..d20d07d 100644 --- a/db/migrate/20220407051821_create_trickest_poc_cves.rb +++ b/db/migrate/20220407051821_create_trickest_poc_cves.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateTrickestPocCves < ActiveRecord::Migration[7.0] def change create_table :trickest_poc_cves do |t| diff --git a/db/migrate/20220407083218_create_cvemon_cves.rb b/db/migrate/20220407083218_create_cvemon_cves.rb index f66528c..86ff807 100644 --- a/db/migrate/20220407083218_create_cvemon_cves.rb +++ b/db/migrate/20220407083218_create_cvemon_cves.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateCvemonCves < ActiveRecord::Migration[7.0] def change create_table :cvemon_cves do |t| diff --git a/db/migrate/20220407223152_create_cnas.rb b/db/migrate/20220407223152_create_cnas.rb index 32c3feb..b7444ce 100644 --- a/db/migrate/20220407223152_create_cnas.rb +++ b/db/migrate/20220407223152_create_cnas.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateCnas < ActiveRecord::Migration[7.0] def change create_table :cnas do |t| diff --git a/db/migrate/20220411174826_create_github_users.rb b/db/migrate/20220411174826_create_github_users.rb index ee9a10b..5be55ea 100644 --- a/db/migrate/20220411174826_create_github_users.rb +++ b/db/migrate/20220411174826_create_github_users.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateGithubUsers < ActiveRecord::Migration[7.0] def change create_table :github_users do |t| diff --git a/db/migrate/20220411181501_create_github_advisories.rb b/db/migrate/20220411181501_create_github_advisories.rb index a3eac95..ffb8b08 100644 --- a/db/migrate/20220411181501_create_github_advisories.rb +++ b/db/migrate/20220411181501_create_github_advisories.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateGithubAdvisories < ActiveRecord::Migration[7.0] def change create_table :github_advisories do |t| diff --git a/db/schema.rb b/db/schema.rb index 0c544d3..ef96235 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This file is auto-generated from the current state of the database. Instead # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. @@ -10,122 +12,121 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do +ActiveRecord::Schema[7.0].define(version: 20_220_411_181_501) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" + enable_extension 'plpgsql' - create_table "cnas", force: :cascade do |t| - t.string "short_name" - t.string "cna_id" - t.string "organization_name" - t.string "scope" - t.jsonb "contact" - t.jsonb "disclosure_policy" - t.jsonb "security_advisories" - t.string "resources", array: true - t.jsonb "cna" - t.string "country" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["cna_id"], name: "index_cnas_on_cna_id", unique: true + create_table 'cnas', force: :cascade do |t| + t.string 'short_name' + t.string 'cna_id' + t.string 'organization_name' + t.string 'scope' + t.jsonb 'contact' + t.jsonb 'disclosure_policy' + t.jsonb 'security_advisories' + t.string 'resources', array: true + t.jsonb 'cna' + t.string 'country' + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['cna_id'], name: 'index_cnas_on_cna_id', unique: true end - create_table "cpes", force: :cascade do |t| - t.string "status" - t.date "modification_date" - t.integer "nvd_id" - t.jsonb "references" - t.string "title" - t.string "name" - t.index ["nvd_id"], name: "index_cpes_on_nvd_id", unique: true + create_table 'cpes', force: :cascade do |t| + t.string 'status' + t.date 'modification_date' + t.integer 'nvd_id' + t.jsonb 'references' + t.string 'title' + t.string 'name' + t.index ['nvd_id'], name: 'index_cpes_on_nvd_id', unique: true end - create_table "cvemon_cves", force: :cascade do |t| - t.string "cve_id" - t.string "urls", array: true - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["cve_id"], name: "index_cvemon_cves_on_cve_id", unique: true + create_table 'cvemon_cves', force: :cascade do |t| + t.string 'cve_id' + t.string 'urls', array: true + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['cve_id'], name: 'index_cvemon_cves_on_cve_id', unique: true end - create_table "cves", force: :cascade do |t| - t.jsonb "cve_data_meta" - t.string "cve_id" - t.jsonb "affects" - t.string "data_format" - t.string "data_type" - t.string "data_version" - t.jsonb "description" - t.jsonb "impact" - t.jsonb "problemtype" - t.jsonb "references" - t.jsonb "source" - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false - t.index ["cve_id"], name: "index_cves_on_cve_id", unique: true + create_table 'cves', force: :cascade do |t| + t.jsonb 'cve_data_meta' + t.string 'cve_id' + t.jsonb 'affects' + t.string 'data_format' + t.string 'data_type' + t.string 'data_version' + t.jsonb 'description' + t.jsonb 'impact' + t.jsonb 'problemtype' + t.jsonb 'references' + t.jsonb 'source' + t.datetime 'created_at', precision: nil, null: false + t.datetime 'updated_at', precision: nil, null: false + t.index ['cve_id'], name: 'index_cves_on_cve_id', unique: true end - create_table "github_advisories", force: :cascade do |t| - t.string "schema_version" - t.string "ghsa_id" - t.date "modified" - t.date "published" - t.string "aliases", array: true - t.string "summary" - t.string "details" - t.jsonb "severity" - t.jsonb "affected" - t.jsonb "references" - t.jsonb "database_specific" - t.index ["ghsa_id"], name: "index_github_advisories_on_ghsa_id", unique: true + create_table 'github_advisories', force: :cascade do |t| + t.string 'schema_version' + t.string 'ghsa_id' + t.date 'modified' + t.date 'published' + t.string 'aliases', array: true + t.string 'summary' + t.string 'details' + t.jsonb 'severity' + t.jsonb 'affected' + t.jsonb 'references' + t.jsonb 'database_specific' + t.index ['ghsa_id'], name: 'index_github_advisories_on_ghsa_id', unique: true end - create_table "github_pocs", force: :cascade do |t| - t.integer "github_poc_id" - t.string "cve_id", default: "None" - t.string "name" - t.string "full_name" - t.jsonb "owner" - t.string "html_url" - t.string "description" - t.boolean "fork" - t.date "created_at" - t.date "updated_at" - t.date "pushed_at" - t.integer "stargazers_count" - t.integer "watchers_count" - t.integer "forks_count" - t.boolean "allow_forking" - t.boolean "is_template" - t.string "topics", array: true - t.string "visibility" - t.integer "forks" - t.integer "watchers" - t.integer "score" - t.index ["github_poc_id"], name: "index_github_pocs_on_github_poc_id", unique: true + create_table 'github_pocs', force: :cascade do |t| + t.integer 'github_poc_id' + t.string 'cve_id', default: 'None' + t.string 'name' + t.string 'full_name' + t.jsonb 'owner' + t.string 'html_url' + t.string 'description' + t.boolean 'fork' + t.date 'created_at' + t.date 'updated_at' + t.date 'pushed_at' + t.integer 'stargazers_count' + t.integer 'watchers_count' + t.integer 'forks_count' + t.boolean 'allow_forking' + t.boolean 'is_template' + t.string 'topics', array: true + t.string 'visibility' + t.integer 'forks' + t.integer 'watchers' + t.integer 'score' + t.index ['github_poc_id'], name: 'index_github_pocs_on_github_poc_id', unique: true end - create_table "github_users", force: :cascade do |t| - t.string "github_id" - t.string "login" - t.string "name" - t.string "avatar_url" - t.string "bio" - t.text "bio_html" - t.string "location" - t.jsonb "repositories" + create_table 'github_users', force: :cascade do |t| + t.string 'github_id' + t.string 'login' + t.string 'name' + t.string 'avatar_url' + t.string 'bio' + t.text 'bio_html' + t.string 'location' + t.jsonb 'repositories' end - create_table "inthewild_cve_exploits", force: :cascade do |t| - t.string "cve_id" - t.string "earliest_report" + create_table 'inthewild_cve_exploits', force: :cascade do |t| + t.string 'cve_id' + t.string 'earliest_report' end - create_table "trickest_poc_cves", force: :cascade do |t| - t.string "cve_id" - t.string "cve_url" - t.string "description" - t.string "poc_links", array: true + create_table 'trickest_poc_cves', force: :cascade do |t| + t.string 'cve_id' + t.string 'cve_url' + t.string 'description' + t.string 'poc_links', array: true end - end diff --git a/db/seeds.rb b/db/seeds.rb index bf2355a..d2c3195 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This file should contain all the record creation needed to seed the database with its default values. # The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup). # @@ -6,75 +8,75 @@ # movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }]) # Character.create(name: 'Luke', movie: movies.first) -require '/data_importer/lib/importers/cpe_importer.rb' -require '/data_importer/lib/importers/cve_list_importer.rb' -require '/data_importer/lib/importers/poc_in_github_importer.rb' -require '/data_importer/lib/importers/inthewild_cve_exploit_importer.rb' -require '/data_importer/lib/importers/trickest_poc_cve_importer.rb' -require '/data_importer/lib/importers/cvemon_cve_importer.rb' -require '/data_importer/lib/importers/cna_importer.rb' -require '/data_importer/lib/importers/github_advisory_importer.rb' -require '/data_importer/lib/importers/github_user_importer.rb' +require '/data_importer/lib/importers/cpe_importer' +require '/data_importer/lib/importers/cve_list_importer' +require '/data_importer/lib/importers/poc_in_github_importer' +require '/data_importer/lib/importers/inthewild_cve_exploit_importer' +require '/data_importer/lib/importers/trickest_poc_cve_importer' +require '/data_importer/lib/importers/cvemon_cve_importer' +require '/data_importer/lib/importers/cna_importer' +require '/data_importer/lib/importers/github_advisory_importer' +require '/data_importer/lib/importers/github_user_importer' def line_sep puts '----------' * 12 end def perform - import_cves - import_github_pocs - import_trickest_poc_cves - import_inthewild_cve_exploits - import_cvemon_cves - import_cpes - import_cnas - import_github_advisories - import_github_usernames + import_cves + import_github_pocs + import_trickest_poc_cves + import_inthewild_cve_exploits + import_cvemon_cves + import_cpes + import_cnas + import_github_advisories + import_github_usernames end def import_cves - line_sep - CveListImporter.new.import + line_sep + CveListImporter.new.import end def import_cpes - line_sep - CpeImporter.download_and_import + line_sep + CpeImporter.download_and_import end def import_github_pocs - line_sep - PocInGithubImporter.new.import + line_sep + PocInGithubImporter.new.import end def import_github_advisories - line_sep - GithubAdvisoryImporter.new.import + line_sep + GithubAdvisoryImporter.new.import end def import_github_usernames - line_sep - GithubUserImporter.new.import + line_sep + GithubUserImporter.new.import end def import_inthewild_cve_exploits - line_sep - InthewildCveExploitImporter.new.import + line_sep + InthewildCveExploitImporter.new.import end def import_trickest_poc_cves - line_sep - TrickestPocCveImporter.new.import + line_sep + TrickestPocCveImporter.new.import end def import_cvemon_cves - line_sep - CvemonCveImporter.new.import + line_sep + CvemonCveImporter.new.import end def import_cnas - line_sep - CnaImporter.new.import + line_sep + CnaImporter.new.import end -perform \ No newline at end of file +perform diff --git a/lib/github_api/github_api.rb b/lib/github_api/github_api.rb index 013193d..1e3db94 100644 --- a/lib/github_api/github_api.rb +++ b/lib/github_api/github_api.rb @@ -1,18 +1,20 @@ -require "graphql/client" -require "graphql/client/http" +# frozen_string_literal: true + +require 'graphql/client' +require 'graphql/client/http' module GithubApi GITHUB_ACCESS_TOKEN = ENV['github_api_token'] URL = 'https://api.github.com/graphql' HttpAdapter = GraphQL::Client::HTTP.new(URL) do - def headers(context) + def headers(_context) { - "Authorization" => "Bearer #{GITHUB_ACCESS_TOKEN}", - "User-Agent" => 'Ruby' + 'Authorization' => "Bearer #{GITHUB_ACCESS_TOKEN}", + 'User-Agent' => 'Ruby' } end end Schema = GraphQL::Client.load_schema(HttpAdapter) Client = GraphQL::Client.new(schema: Schema, execute: HttpAdapter) -end \ No newline at end of file +end diff --git a/lib/github_api/owner_repos.rb b/lib/github_api/owner_repos.rb index 715b263..6d9dc10 100644 --- a/lib/github_api/owner_repos.rb +++ b/lib/github_api/owner_repos.rb @@ -1,8 +1,10 @@ -require '/data_importer/lib/github_api/github_api.rb' +# frozen_string_literal: true + +require '/data_importer/lib/github_api/github_api' module GithubApi - class OwnerRepos - OwnerReposQuery = GithubApi::Client.parse <<-'GRAPHQL' + class OwnerRepos + OwnerReposQuery = GithubApi::Client.parse <<-'GRAPHQL' query($owner: String!, $cursor: String) { repositoryOwner(login: $owner) { repositories( @@ -53,19 +55,19 @@ module GithubApi } } } - GRAPHQL - - def self.find(username) - #Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do - response = GithubApi::Client.query(OwnerReposQuery, variables: { owner: username }) - if response.errors.any? - raise QueryExecutionError.new(response.errors[:data].join(", ")) - else - response.data.repository_owner.repositories.nodes.map(&:to_h) - end - #end - end - end + GRAPHQL + + def self.find(username) + # Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do + response = GithubApi::Client.query(OwnerReposQuery, variables: { owner: username }) + if response.errors.any? + raise QueryExecutionError, response.errors[:data].join(', ') + else + response.data.repository_owner.repositories.nodes.map(&:to_h) + end + # end + end + end end -class QueryExecutionError < StandardError; end \ No newline at end of file +class QueryExecutionError < StandardError; end diff --git a/lib/github_api/security_advisory.rb b/lib/github_api/security_advisory.rb index 23496a5..6e3e8b8 100644 --- a/lib/github_api/security_advisory.rb +++ b/lib/github_api/security_advisory.rb @@ -1,6 +1,8 @@ -require '/data_importer/lib/github_api/github_api.rb' +# frozen_string_literal: true + +require '/data_importer/lib/github_api/github_api' module GithubApi -class SecurityAdvisory + class SecurityAdvisory SecurityAdvisoryQuery = GithubApi::Client.parse <<-'GRAPHQL' query($ghsa_id: String!) { securityAdvisory(ghsaId: $ghsa_id) { @@ -45,16 +47,16 @@ class SecurityAdvisory GRAPHQL def self.find(ghsa_id) - #Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do - response = GithubApi::Client.query(SecurityAdvisoryQuery, variables: { ghsa_id: ghsa_id }) - if response.errors.any? - raise QueryExecutionError.new(response.errors[:data].join(", ")) - else - response.data.security_advisory - end - #end + # Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do + response = GithubApi::Client.query(SecurityAdvisoryQuery, variables: { ghsa_id: ghsa_id }) + if response.errors.any? + raise QueryExecutionError, response.errors[:data].join(', ') + else + response.data.security_advisory + end + # end end -end + end end -class QueryExecutionError < StandardError; end \ No newline at end of file +class QueryExecutionError < StandardError; end diff --git a/lib/github_api/user.rb b/lib/github_api/user.rb index 1593bd6..81818a3 100644 --- a/lib/github_api/user.rb +++ b/lib/github_api/user.rb @@ -1,6 +1,8 @@ -require '/data_importer/lib/github_api/github_api.rb' +# frozen_string_literal: true + +require '/data_importer/lib/github_api/github_api' module GithubApi -class User + class User UserProfileQuery = GithubApi::Client.parse <<-'GRAPHQL' query($username: String!) { user(login: $username) { @@ -16,16 +18,16 @@ class User GRAPHQL def self.find(username) - #Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do - response = GithubApi::Client.query(UserProfileQuery, variables: { username: username }) - if response.errors.any? - raise QueryExecutionError.new(response.errors[:data].join(", ")) - else - response.data.user - end - #end + # Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do + response = GithubApi::Client.query(UserProfileQuery, variables: { username: username }) + if response.errors.any? + raise QueryExecutionError, response.errors[:data].join(', ') + else + response.data.user + end + # end end -end + end end -class QueryExecutionError < StandardError; end \ No newline at end of file +class QueryExecutionError < StandardError; end diff --git a/lib/importers/cna_importer.rb b/lib/importers/cna_importer.rb index 4f35d6c..b33f617 100644 --- a/lib/importers/cna_importer.rb +++ b/lib/importers/cna_importer.rb @@ -1,25 +1,27 @@ +# frozen_string_literal: true + require 'json' -require '/data_importer/lib/json_helper.rb' +require '/data_importer/lib/json_helper' require 'rest-client' class CnaImporter - - EXPECTED_KEYS = [ - :short_name, - :cna_id, - :organization_name, - :scope, - :contact, - :disclosure_policy, - :security_advisories, - :resources, - :cna, - :country + EXPECTED_KEYS = %i[ + short_name + cna_id + organization_name + scope + contact + disclosure_policy + security_advisories + resources + cna + country ].freeze - EMPTY_HASH = EXPECTED_KEYS.map {|k| [k, nil] }.to_h.freeze + EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze attr_accessor :url + def initialize @url = 'https://raw.githubusercontent.com/CVEProject/cve-website/dev/src/assets/data/CNAsList.json' end @@ -47,9 +49,8 @@ class CnaImporter def import jsons = get_json - merged_hashes = jsons.map {|h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) } - puts "Now importing CNAs." + merged_hashes = jsons.map { |h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) } + puts 'Now importing CNAs.' Cna.upsert_all(merged_hashes, unique_by: :cna_id) end - end diff --git a/lib/importers/cpe_importer.rb b/lib/importers/cpe_importer.rb index 59b0360..28d3515 100644 --- a/lib/importers/cpe_importer.rb +++ b/lib/importers/cpe_importer.rb @@ -48,8 +48,8 @@ class CpeImporter node.name == 'cpe-item' && node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT end - def self.import(bulk_count = 20000, filepath = '/data_importer/data/official-cpe-dictionary_v2.2.xml.gz') - puts "Now importing Cpes." + def self.import(bulk_count = 20_000, filepath = '/data_importer/data/official-cpe-dictionary_v2.2.xml.gz') + puts 'Now importing Cpes.' Zlib::GzipReader.open(filepath) do |file| items = [] Nokogiri::XML::Reader.from_io(file).each do |node| diff --git a/lib/importers/cve_list_importer.rb b/lib/importers/cve_list_importer.rb index dce6e1b..f5d4bfa 100644 --- a/lib/importers/cve_list_importer.rb +++ b/lib/importers/cve_list_importer.rb @@ -1,29 +1,31 @@ +# frozen_string_literal: true + require 'git' require 'json' require 'date' -require '/data_importer/lib/importers/github_repo.rb' -require '/data_importer/lib/json_helper.rb' +require '/data_importer/lib/importers/github_repo' +require '/data_importer/lib/json_helper' # This class can be used to import cvelist json data from mitre from their github repo class CveListImporter < GithubRepo - EXPECTED_KEYS = [ - :cve_data_meta, - :cve_id, - :affects, - :data_format, - :data_type, - :data_version, - :description, - :impact, - :problemtype, - :references, - :source -].freeze + EXPECTED_KEYS = %i[ + cve_data_meta + cve_id + affects + data_format + data_type + data_version + description + impact + problemtype + references + source + ].freeze -EMPTY_HASH = EXPECTED_KEYS.map {|k| [k, nil] }.to_h.freeze + EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze def initialize - super(repo_url='https://github.com/CVEProject/cvelist.git', repo_path='/data_importer/data/cve_list') + super(repo_url = 'https://github.com/CVEProject/cvelist.git', repo_path = '/data_importer/data/cve_list') end def list_jsons_for_year(year) @@ -33,12 +35,12 @@ EMPTY_HASH = EXPECTED_KEYS.map {|k| [k, nil] }.to_h.freeze def read_jsons_for_year(year) filenames = list_jsons_for_year(year) - hashes = filenames.map do |filename| + hashes = filenames.map do |filename| json = read_json(filename) - json_transformed = JsonHelper.deep_transform_keys(json) + json_transformed = JsonHelper.deep_transform_keys(json) add_cve_id_to_json_key(json_transformed) end - hashes.map {|h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) } + hashes.map { |h| h.slice(*EXPECTED_KEYS).reverse_merge(EMPTY_HASH) } end def add_cve_id_to_json_key(json) @@ -53,14 +55,14 @@ EMPTY_HASH = EXPECTED_KEYS.map {|k| [k, nil] }.to_h.freeze (1999..Date.today.year).map do |year| cves = read_jsons_for_year(year) - #ids = cves.map { |cve| cve[:cve_id] } - #cve_ids_in_db = Cve.where(:cve_id => ids).pluck(:cve_id) + # ids = cves.map { |cve| cve[:cve_id] } + # cve_ids_in_db = Cve.where(:cve_id => ids).pluck(:cve_id) - #new_cve_ids = ids - cve_ids_in_db - #new_cves = cves.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } + # new_cve_ids = ids - cve_ids_in_db + # new_cves = cves.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } puts "Importing any new CVEs from #{year}" Cve.upsert_all(cves, unique_by: :cve_id) end end -end \ No newline at end of file +end diff --git a/lib/importers/cvemon_cve_importer.rb b/lib/importers/cvemon_cve_importer.rb index 4acb8e0..01d314e 100644 --- a/lib/importers/cvemon_cve_importer.rb +++ b/lib/importers/cvemon_cve_importer.rb @@ -1,32 +1,35 @@ +# frozen_string_literal: true + require 'rest-client' require 'json' class CvemonCveImporter - attr_accessor :url - def initialize - @url = 'https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json' - end + attr_accessor :url - def get_cve_data - r = RestClient::Request.execute( - :method => :get, - :url => url, - :headers => {"Content-type": "application/json"} - ) - if r.code == 200 - JSON.parse(r.body) - else - puts "HTTP Code #{r.code}" - end - end + def initialize + @url = 'https://raw.githubusercontent.com/ARPSyndicate/cvemon/main/data.json' + end - def import - feed = get_cve_data - cve_ids = feed.keys - puts "Now importing CvemonCves." - cves = cve_ids.map do |cve_id| - { :cve_id => cve_id, :urls => feed[cve_id] } - end - CvemonCve.upsert_all(cves, unique_by: :cve_id) + def get_cve_data + r = RestClient::Request.execute( + method: :get, + url: url, + headers: { "Content-type": 'application/json' } + ) + if r.code == 200 + JSON.parse(r.body) + else + puts "HTTP Code #{r.code}" end -end \ No newline at end of file + end + + def import + feed = get_cve_data + cve_ids = feed.keys + puts 'Now importing CvemonCves.' + cves = cve_ids.map do |cve_id| + { cve_id: cve_id, urls: feed[cve_id] } + end + CvemonCve.upsert_all(cves, unique_by: :cve_id) + end +end diff --git a/lib/importers/github_advisory_importer.rb b/lib/importers/github_advisory_importer.rb index f6e7df3..8db565a 100644 --- a/lib/importers/github_advisory_importer.rb +++ b/lib/importers/github_advisory_importer.rb @@ -1,77 +1,79 @@ -require '/data_importer/lib/importers/github_repo.rb' +# frozen_string_literal: true + +require '/data_importer/lib/importers/github_repo' class GithubAdvisoryImporter < GithubRepo - # repo has years that begin with 2017 as first GHSA - YEAR_RANGE = (2017..Date.today.year) + # repo has years that begin with 2017 as first GHSA + YEAR_RANGE = (2017..Date.today.year).freeze - def initialize - super(repo_url='https://github.com/github/advisory-database.git', repo_path='/data_importer/data/github_advisories') + def initialize + super(repo_url = 'https://github.com/github/advisory-database.git', repo_path = '/data_importer/data/github_advisories') + end + + def advisory_paths + advisory_path = "#{repo_path}/advisories" + { + base_path: advisory_path, + github_reviewed_path: "#{advisory_path}/github-reviewed", + unreviewed_path: "#{advisory_path}/unreviewed" + } + end + + def list_jsons_for_year(year) + json_wildcard = '*.json' + github_reviewed_year_fp = "#{advisory_paths[:github_reviewed_path]}/#{year}/*/*" + unreviewed_year_fp = "#{advisory_paths[:unreviewed_path]}/#{year}/*/*" + + github_reviewed_jsons_fp = Dir["#{github_reviewed_year_fp}/#{json_wildcard}"] + unreviewed_jsons_fp = Dir["#{unreviewed_year_fp}/#{json_wildcard}"] + + { + github_reviewed_jsons: github_reviewed_jsons_fp, + unreviewed_jsons: unreviewed_jsons_fp + } + end + + def read_jsons_for_year(year) + fp_hash = list_jsons_for_year(year) + fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons] + jsons = fns.map do |fn| + read_json(fn) end + jsons.flatten + end - def advisory_paths - advisory_path = "#{repo_path}/advisories" - { - :base_path => advisory_path, - :github_reviewed_path => "#{advisory_path}/github-reviewed", - :unreviewed_path => "#{advisory_path}/unreviewed" - } - end + def attrs_from_item(json) + attrs = {} + attrs[:schema_version] = json['schema_version'] + attrs[:ghsa_id] = json['id'] + attrs[:modified] = json['modified'] + attrs[:published] = json['published'] + attrs[:aliases] = json['aliases'] + attrs[:summary] = json['summary'] + attrs[:details] = json['details'] + attrs[:severity] = json['severity'] + attrs[:affected] = json['affected'] + attrs[:references] = json['references'] + attrs[:database_specific] = json['database_specific'] + attrs + end - def list_jsons_for_year(year) - json_wildcard = "*.json" - github_reviewed_year_fp = "#{advisory_paths[:github_reviewed_path]}/#{year}/*/*" - unreviewed_year_fp = "#{advisory_paths[:unreviewed_path]}/#{year}/*/*" - - github_reviewed_jsons_fp = Dir["#{github_reviewed_year_fp}/#{json_wildcard}"] - unreviewed_jsons_fp = Dir["#{unreviewed_year_fp}/#{json_wildcard}"] - - { - :github_reviewed_jsons => github_reviewed_jsons_fp, - :unreviewed_jsons => unreviewed_jsons_fp - } + def bulk_insert(jsons) + GithubAdvisory.bulk_insert do |worker| + jsons.each do |json| + attrs = attrs_from_item(json) + worker.add(attrs) end - - def read_jsons_for_year(year) - fp_hash = list_jsons_for_year(year) - fns = fp_hash[:github_reviewed_jsons] + fp_hash[:unreviewed_jsons] - jsons = fns.map do |fn| - read_json(fn) - end - jsons.flatten - end - - def attrs_from_item(json) - attrs = {} - attrs[:schema_version] = json['schema_version'] - attrs[:ghsa_id] = json['id'] - attrs[:modified] = json['modified'] - attrs[:published] = json['published'] - attrs[:aliases] = json['aliases'] - attrs[:summary] = json['summary'] - attrs[:details] = json['details'] - attrs[:severity] = json['severity'] - attrs[:affected] = json['affected'] - attrs[:references] = json['references'] - attrs[:database_specific] = json['database_specific'] - attrs end + end - def bulk_insert(jsons) - GithubAdvisory.bulk_insert do |worker| - jsons.each do |json| - attrs = attrs_from_item(json) - worker.add(attrs) - end - end + def import + pull_or_clone + puts 'Now importing GithubAdvisories.' + YEAR_RANGE.each do |year| + puts "Importing advisory data from #{year}" + jsons = read_jsons_for_year(year) + bulk_insert(jsons) end - - def import - pull_or_clone - puts "Now importing GithubAdvisories." - YEAR_RANGE.each do |year| - puts "Importing advisory data from #{year}" - jsons = read_jsons_for_year(year) - bulk_insert(jsons) - end - end -end \ No newline at end of file + end +end diff --git a/lib/importers/github_repo.rb b/lib/importers/github_repo.rb index 17370c0..76e6430 100644 --- a/lib/importers/github_repo.rb +++ b/lib/importers/github_repo.rb @@ -1,33 +1,35 @@ +# frozen_string_literal: true + class GithubRepo - attr_accessor :repo_url, :repo_path + attr_accessor :repo_url, :repo_path - def initialize(repo_url=nil, repo_path=nil) - @repo_url = repo_url - @repo_path = repo_path - end - - def git_clone_repo - if repo_url.nil? || repo_path.nil? - puts "Please provide a repo url and repo_path" - else - Git.clone(repo_url, repo_path) - end - end - - def pull_latest_changes - `cd #{repo_path}; git pull;` - puts "Now pulling latest changes from #{repo_path}" - end - - def read_json(filename) - JSON.parse(File.read(filename), symbolize_names: true) - end + def initialize(repo_url = nil, repo_path = nil) + @repo_url = repo_url + @repo_path = repo_path + end - def pull_or_clone - if Dir.exist?(repo_path) - pull_latest_changes - else - git_clone_repo - end + def git_clone_repo + if repo_url.nil? || repo_path.nil? + puts 'Please provide a repo url and repo_path' + else + Git.clone(repo_url, repo_path) end -end \ No newline at end of file + end + + def pull_latest_changes + `cd #{repo_path}; git pull;` + puts "Now pulling latest changes from #{repo_path}" + end + + def read_json(filename) + JSON.parse(File.read(filename), symbolize_names: true) + end + + def pull_or_clone + if Dir.exist?(repo_path) + pull_latest_changes + else + git_clone_repo + end + end +end diff --git a/lib/importers/github_user_importer.rb b/lib/importers/github_user_importer.rb index 5a1d3ad..352daad 100644 --- a/lib/importers/github_user_importer.rb +++ b/lib/importers/github_user_importer.rb @@ -1,8 +1,11 @@ -require '/data_importer/lib/github_api/user.rb' -require '/data_importer/lib/github_api/owner_repos.rb' +# frozen_string_literal: true + +require '/data_importer/lib/github_api/user' +require '/data_importer/lib/github_api/owner_repos' class GithubUserImporter attr_accessor :filepath, :usernames + def initialize @filepath = '/data_importer/data/github_usernames.txt' @usernames = File.read(filepath).split("\n") @@ -10,7 +13,7 @@ class GithubUserImporter def username_hashes usernames.map do |username| - Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: lambda { |n| 4**n } ) do + Retryable.retryable(tries: 3, on: QueryExecutionError, sleep: ->(n) { 4**n }) do username_response = GithubApi::User.find(username) username_repos = GithubApi::OwnerRepos.find(username) repos_hash = { 'repositories' => username_repos } @@ -43,10 +46,10 @@ class GithubUserImporter def import if filepath.nil? - puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line." + puts 'Please provide a filepath in the projects data dir named github_usernames.txt with one username per line.' else - puts "Now importing GithubUsers" - usernames = username_hashes.map {|h| h['login'] } + puts 'Now importing GithubUsers' + usernames = username_hashes.map { |h| h['login'] } puts "Now importing data from the following usernames: #{usernames}" bulk_insert(username_hashes) end diff --git a/lib/importers/inthewild_cve_exploit_importer.rb b/lib/importers/inthewild_cve_exploit_importer.rb index e126c6c..a428daa 100644 --- a/lib/importers/inthewild_cve_exploit_importer.rb +++ b/lib/importers/inthewild_cve_exploit_importer.rb @@ -1,47 +1,50 @@ +# frozen_string_literal: true + require 'bulk_insert' require 'json' class InthewildCveExploitImporter - attr_accessor :url - def initialize - @url = 'https://inthewild.io/api/exploited' + attr_accessor :url + + def initialize + @url = 'https://inthewild.io/api/exploited' + end + + def get_exploit_feed + r = RestClient::Request.execute( + method: :get, + url: url, + headers: { "Content-type": 'application/json' } + ) + if r.code == 200 + JSON.parse(r.body) + else + puts "HTTP Code #{r.code}" + end + end + + def cve_attrs_from_item(json) + cve_attrs = {} + cve_attrs[:cve_id] = json['id'] + cve_attrs[:earliest_report] = json['earliestReport'] + cve_attrs + end + + def bulk_insert(cves) + InthewildCveExploit.bulk_insert do |worker| + cves.each do |attrs| + worker.add(attrs) + end + end + end + + def import + feed = get_exploit_feed + puts 'Now importing InthewildCveExploits.' + cves = feed.map do |cve_entry| + cve_attrs_from_item(cve_entry) end - def get_exploit_feed - r = RestClient::Request.execute( - :method => :get, - :url => url, - :headers => {"Content-type": "application/json"} - ) - if r.code == 200 - JSON.parse(r.body) - else - puts "HTTP Code #{r.code}" - end - end - - def cve_attrs_from_item(json) - cve_attrs = {} - cve_attrs[:cve_id] = json['id'] - cve_attrs[:earliest_report] = json['earliestReport'] - cve_attrs - end - - def bulk_insert(cves) - InthewildCveExploit.bulk_insert do |worker| - cves.each do |attrs| - worker.add(attrs) - end - end - end - - def import - feed = get_exploit_feed - puts "Now importing InthewildCveExploits." - cves = feed.map do |cve_entry| - cve_attrs_from_item(cve_entry) - end - - bulk_insert(cves) - end -end \ No newline at end of file + bulk_insert(cves) + end +end diff --git a/lib/importers/poc_in_github_importer.rb b/lib/importers/poc_in_github_importer.rb index e0f5cb6..82c61db 100644 --- a/lib/importers/poc_in_github_importer.rb +++ b/lib/importers/poc_in_github_importer.rb @@ -1,19 +1,21 @@ +# frozen_string_literal: true + require 'git' require 'json' require 'date' require 'bulk_insert' -require '/data_importer/lib/importers/github_repo.rb' +require '/data_importer/lib/importers/github_repo' class PocInGithubImporter < GithubRepo - CVE_MATCHER = /(CVE|cve)-\d{4}-\d{4,7}/ + CVE_MATCHER = /(CVE|cve)-\d{4}-\d{4,7}/.freeze def initialize - super(repo_url='https://github.com/nomi-sec/PoC-in-GitHub.git', repo_path='/data_importer/data/poc_in_github') + super(repo_url = 'https://github.com/nomi-sec/PoC-in-GitHub.git', repo_path = '/data_importer/data/poc_in_github') end # all the files are named CVE-year-1234.json in this repo def cve_from_filename(filename) - File.basename(filename,File.extname(filename)) + File.basename(filename, File.extname(filename)) end # regex extract substring thats a cve-id from either the name or full_name json entries @@ -22,10 +24,9 @@ class PocInGithubImporter < GithubRepo fullname = json['full_name'] description = json['description'] id = name.match(CVE_MATCHER)[0] || fullname.match(CVE_MATCHER)[0] || description.match(CVE_MATCHER)[0] - debug_hash = {:name => name, :fullname => fullname, :description => description, :id => id.upcase } + debug_hash = { name: name, fullname: fullname, description: description, id: id.upcase } puts debug_hash - cve_id = id.upcase - cve_id + id.upcase end def list_jsons_for_year(year) @@ -36,7 +37,7 @@ class PocInGithubImporter < GithubRepo def read_jsons_for_year(year) filenames = list_jsons_for_year(year) filenames.map do |f| - {:cve_id => cve_from_filename(f), :file_data => read_json(f) } + { cve_id: cve_from_filename(f), file_data: read_json(f) } end.flatten end @@ -46,7 +47,7 @@ class PocInGithubImporter < GithubRepo end end - def cve_attrs_from_item(json, cve_attrs={}) + def cve_attrs_from_item(json, cve_attrs = {}) cve_attrs[:github_poc_id] = json['id'] cve_attrs[:name] = json['name'] cve_attrs[:full_name] = json['full_name'] @@ -73,13 +74,13 @@ class PocInGithubImporter < GithubRepo # for bulk inserting def cves_for_year(year) json_data = read_jsons_for_year(year) - + json_data.map do |info_hash| cve_id = info_hash[:cve_id] json_fd = info_hash[:file_data] json_fd.map do |entry| - cve_attrs_from_item(entry, cve_attrs={:cve_id => cve_id}) + cve_attrs_from_item(entry, cve_attrs = { cve_id: cve_id }) end end.flatten end @@ -92,7 +93,7 @@ class PocInGithubImporter < GithubRepo cves_from_json = cves_for_year(year) ids = cves_from_json.map { |cve| cve[:github_poc_id] } - ids_in_db = GithubPoc.where(:github_poc_id => ids).pluck(:github_poc_id) + ids_in_db = GithubPoc.where(github_poc_id: ids).pluck(:github_poc_id) new_ids = ids - ids_in_db new_cves = cves_from_json.select { |cve| cve if new_ids.include?(cve[:github_poc_id]) } @@ -108,4 +109,4 @@ class PocInGithubImporter < GithubRepo end end end -end \ No newline at end of file +end diff --git a/lib/importers/trickest_poc_cve_importer.rb b/lib/importers/trickest_poc_cve_importer.rb index 8f5d58b..bc29887 100644 --- a/lib/importers/trickest_poc_cve_importer.rb +++ b/lib/importers/trickest_poc_cve_importer.rb @@ -1,126 +1,128 @@ +# frozen_string_literal: true + require 'git' require 'json' require 'date' require 'bulk_insert' class TrickestPocCveImporter - attr_accessor :repo_url, :repo_path + attr_accessor :repo_url, :repo_path - def initialize - @repo_url = 'https://github.com/trickest/cve.git' - @repo_path = '/data_importer/data/trickest_cve' + def initialize + @repo_url = 'https://github.com/trickest/cve.git' + @repo_path = '/data_importer/data/trickest_cve' + end + + def git_clone_repo + Git.clone(repo_url, repo_path) + end + + def pull_latest_changes + `cd #{repo_path}; git pull;` + puts "Now pulling latest changes from #{repo_path}" + end + + def pull_or_clone(repo_path) + if Dir.exist?(repo_path) + pull_latest_changes + else + git_clone_repo end - - def git_clone_repo - Git.clone(repo_url, repo_path) + end + + def read_markdown(filename) + data = File.read(filename) + formatter = RDoc::Markup::ToHtml.new(RDoc::Options.new, nil) + # should give us the html doc + RDoc::Markdown.parse(data).accept(formatter) + end + + def html_to_hash(html) + data_hash = {} + doc = Nokogiri::HTML5.parse(html) + h3_nodes = doc.xpath('//h3') + h3_keys = doc.xpath('//h3').map { |n| n.children.first.text } + h4_keys = doc.xpath('//h4').map { |n| n.children.first.text } + data_hash_keys = (h3_keys + h4_keys).flatten + + # cve id is always the first url in the markdown doc + cve_url = doc.xpath('//h3/a').attribute('href').value + cve_id = h3_keys.first + + p_text = doc.xpath('//p').map(&:text) + links_for_poc = doc.xpath('//p/a').map(&:values).flatten + + data_hash['cve_id'] = cve_id + data_hash['cve_url'] = cve_url + # p_text[0] is always an ' '. + data_hash['description'] = p_text[1] + + data_hash['poc_links'] = links_for_poc + data_hash + end + + def list_mds_for_year(year) + year_fp = "#{repo_path}/#{year}" + Dir["#{year_fp}/*.md"] + end + + def read_mds_for_year(year) + filenames = list_mds_for_year(year) + filenames.map { |filename| read_markdown(filename) } + end + + def read_all_mds + (1999..Date.today.year).map do |year| + read_mds_for_year(year.to_s) end - - def pull_latest_changes - `cd #{repo_path}; git pull;` - puts "Now pulling latest changes from #{repo_path}" + end + + def cve_attrs_from_item(json) + cve_attrs = {} + cve_attrs[:cve_id] = json['cve_id'] + cve_attrs[:cve_url] = json['cve_url'] + cve_attrs[:description] = json['description'] + cve_attrs[:poc_links] = json['poc_links'] + cve_attrs + end + + # for bulk inserting + def cves_for_year(year) + htmls = read_mds_for_year(year) + htmls.map do |html| + data_hash = html_to_hash(html) + cve_attrs_from_item(data_hash) + end + end + + def import + if Dir.exist?(repo_path) + pull_latest_changes + else + git_clone_repo end - def pull_or_clone(repo_path) - if Dir.exist?(repo_path) - pull_latest_changes - else - git_clone_repo - end - end - - def read_markdown(filename) - data = File.read(filename) - formatter = RDoc::Markup::ToHtml.new(RDoc::Options.new, nil) - # should give us the html doc - RDoc::Markdown.parse(data).accept(formatter) - end + puts "Now starting import for #{repo_url}." + puts '----------' * 12 + (1999..Date.today.year).map do |year| + cves_from_markdown = cves_for_year(year) - def html_to_hash(html) - data_hash = {} - doc = Nokogiri::HTML5.parse(html) - h3_nodes = doc.xpath('//h3') - h3_keys = doc.xpath('//h3').map {|n| n.children.first.text} - h4_keys = doc.xpath('//h4').map {|n| n.children.first.text} - data_hash_keys = (h3_keys + h4_keys).flatten + ids = cves_from_markdown.map { |cve| cve[:cve_id] } + cve_ids_in_db = TrickestPocCve.where(cve_id: ids).pluck(:cve_id) - # cve id is always the first url in the markdown doc - cve_url = doc.xpath("//h3/a").attribute('href').value - cve_id = h3_keys.first + new_cve_ids = ids - cve_ids_in_db + new_cves = cves_from_markdown.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } + puts "Importing any new CVEs from #{year}" - p_text = doc.xpath('//p').map {|p| p.text } - links_for_poc = doc.xpath('//p/a').map {|a| a.values}.flatten - - data_hash['cve_id'] = cve_id - data_hash['cve_url'] = cve_url - # p_text[0] is always an ' '. - data_hash['description'] = p_text[1] + bulk_insert(new_cves) + end + end - data_hash['poc_links'] = links_for_poc - data_hash - end - - def list_mds_for_year(year) - year_fp = "#{repo_path}/#{year}" - Dir["#{year_fp}/*.md"] - end - - def read_mds_for_year(year) - filenames = list_mds_for_year(year) - filenames.map { |filename| read_markdown(filename) } - end - - def read_all_mds - (1999..Date.today.year).map do |year| - read_mds_for_year(year.to_s) + def bulk_insert(cves) + TrickestPocCve.bulk_insert do |worker| + cves.each do |attrs| + worker.add(attrs) end end - - def cve_attrs_from_item(json) - cve_attrs = {} - cve_attrs[:cve_id] = json['cve_id'] - cve_attrs[:cve_url] = json['cve_url'] - cve_attrs[:description] = json['description'] - cve_attrs[:poc_links] = json['poc_links'] - cve_attrs - end - - # for bulk inserting - def cves_for_year(year) - htmls = read_mds_for_year(year) - htmls.map do |html| - data_hash = html_to_hash(html) - cve_attrs_from_item(data_hash) - end - end - - def import - if Dir.exist?(repo_path) - pull_latest_changes - else - git_clone_repo - end - - puts "Now starting import for #{repo_url}." - puts '----------' * 12 - (1999..Date.today.year).map do |year| - cves_from_markdown = cves_for_year(year) - - ids = cves_from_markdown.map { |cve| cve[:cve_id] } - cve_ids_in_db = TrickestPocCve.where(:cve_id => ids).pluck(:cve_id) - - new_cve_ids = ids - cve_ids_in_db - new_cves = cves_from_markdown.select { |cve| cve if new_cve_ids.include?(cve[:cve_id]) } - puts "Importing any new CVEs from #{year}" - - bulk_insert(new_cves) - end - end - - def bulk_insert(cves) - TrickestPocCve.bulk_insert do |worker| - cves.each do |attrs| - worker.add(attrs) - end - end - end -end \ No newline at end of file + end +end diff --git a/lib/json_helper.rb b/lib/json_helper.rb index 61281dd..98922b8 100644 --- a/lib/json_helper.rb +++ b/lib/json_helper.rb @@ -1,15 +1,15 @@ +# frozen_string_literal: true + class JsonHelper - - def self.deep_transform_keys(json_hash) if json_hash.is_a? Array - json_hash.map {|jh| symbolize_names_snake_case(jh) } + json_hash.map { |jh| symbolize_names_snake_case(jh) } else symbolize_names_snake_case(json_hash) end end def self.symbolize_names_snake_case(json_hash) - json_hash.deep_transform_keys {|k| k.to_s.underscore.to_sym } + json_hash.deep_transform_keys { |k| k.to_s.underscore.to_sym } end end diff --git a/lib/tasks/github_usernames_populate.rb b/lib/tasks/github_usernames_populate.rb index ca8021b..5e9d407 100644 --- a/lib/tasks/github_usernames_populate.rb +++ b/lib/tasks/github_usernames_populate.rb @@ -1,11 +1,13 @@ +# frozen_string_literal: true + class GithubUsernamesPopulate def self.usernames_from_pocs - GithubPoc.pluck(:owner).map {|h| h['login']}.sort.uniq + GithubPoc.pluck(:owner).map { |h| h['login'] }.sort.uniq end def self.update_file fp = '/data_importer/data/github_usernames.txt' - File.open(fp, "w+") do |f| + File.open(fp, 'w+') do |f| puts "Updating username file found at #{fp}" f.puts(usernames_from_pocs) end diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb index d19212a..652febb 100644 --- a/test/application_system_test_case.rb +++ b/test/application_system_test_case.rb @@ -1,4 +1,6 @@ -require "test_helper" +# frozen_string_literal: true + +require 'test_helper' class ApplicationSystemTestCase < ActionDispatch::SystemTestCase driven_by :selenium, using: :chrome, screen_size: [1400, 1400] diff --git a/test/controllers/cpes_controller_test.rb b/test/controllers/cpes_controller_test.rb index ef52cd7..cf9001c 100644 --- a/test/controllers/cpes_controller_test.rb +++ b/test/controllers/cpes_controller_test.rb @@ -1,4 +1,6 @@ -require "test_helper" +# frozen_string_literal: true + +require 'test_helper' class CpesControllerTest < ActionDispatch::IntegrationTest # test "the truth" do diff --git a/test/controllers/cves_controller_test.rb b/test/controllers/cves_controller_test.rb index e03181e..b3b6d62 100644 --- a/test/controllers/cves_controller_test.rb +++ b/test/controllers/cves_controller_test.rb @@ -1,4 +1,6 @@ -require "test_helper" +# frozen_string_literal: true + +require 'test_helper' class CvesControllerTest < ActionDispatch::IntegrationTest # test "the truth" do diff --git a/test/test_helper.rb b/test/test_helper.rb index 3ab84e3..3f27318 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,10 +1,14 @@ +# frozen_string_literal: true + ENV['RAILS_ENV'] ||= 'test' require_relative '../config/environment' require 'rails/test_help' -class ActiveSupport::TestCase - # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. - fixtures :all +module ActiveSupport + class TestCase + # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. + fixtures :all - # Add more helper methods to be used by all tests here... + # Add more helper methods to be used by all tests here... + end end