if you create a text file at ./data/github_usernames.txt with one username per line GithubUserImporter will use this data to import a GithubUser for each one

This commit is contained in:
Brendan McDevitt 2022-04-11 20:53:20 -05:00
parent db6afbfd59
commit d26395def2
15 changed files with 93 additions and 1 deletions

View file

@ -62,6 +62,14 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
get "/github_advisories/:ghsa_id", to: "github_advisories#show"
```
#### GithubUsers
Create a text file named `./data/github_usernames.txt` with one username per line
There is a seed task that will read this file and perform an API call to github API and store the data in DB for each user.
```
get "/github_users", to: "github_users#index"
get "/github_users/:username", to: "github_users#show"
```
#### GithubPocs
```
get "/github_pocs", to: "github_pocs#index"

View file

@ -1,6 +1,7 @@
class CnasController < ApplicationController
def index
@cnas = Cna.all
render json: @cnas.to_json
end
def show

View file

@ -1,6 +1,7 @@
class CpesController < ApplicationController
def index
@cpes = Cpe.all
render json: @cpes.to_json
end
def show

View file

@ -1,6 +1,7 @@
class CvemonCvesController < ApplicationController
def index
@pocs = CvemonCve.all
render json: @pocs.to_json
end
def show

View file

@ -1,6 +1,7 @@
class CvesController < ApplicationController
def index
@cves = Cve.all
render json: @cves.to_json
end
def show

View file

@ -1,10 +1,11 @@
class GithubAdvisoriesController < ApplicationController
def index
@advisories = GithubAdvisory.all
render json: @advisories.to_json
end
def show
@advisory = GithubAdivsory.find_by_ghsa_id(params[:ghsa_id])
@advisory = GithubAdvisory.find_by_ghsa_id(params[:ghsa_id])
render json: @advisory.to_json
end
end

View file

@ -1,6 +1,7 @@
class GithubPocsController < ApplicationController
def index
@pocs = GithubPoc.all
render json: @pocs.to_json
end
def show

View file

@ -0,0 +1,11 @@
class GithubUsersController < ApplicationController
def index
@users = GithubUser.all
render json: @users.to_json
end
def show
@user = GithubUser.find_by_username(params[:username])
render json: @user.to_json
end
end

View file

@ -1,6 +1,7 @@
class TrickestPocCvesController < ApplicationController
def index
@pocs = TrickestPocCve.all
render json: @pocs.to_json
end
def show

View file

@ -0,0 +1,5 @@
class GithubUser< ActiveRecord::Base
def self.find_by_username(username)
find_by(login: username)
end
end

View file

@ -0,0 +1 @@
<h1>Cpes#index</h1>

View file

@ -0,0 +1,2 @@
@users

View file

@ -33,4 +33,7 @@ Rails.application.routes.draw do
get "/github_advisories", to: "github_advisories#index"
get "/github_advisories/:ghsa_id", to: "github_advisories#show"
get "/github_users", to: "github_users#index"
get "/github_users/:username", to: "github_users#show"
end

View file

@ -14,6 +14,7 @@ require '/data_importer/lib/importers/trickest_poc_cve_importer.rb'
require '/data_importer/lib/importers/cvemon_cve_importer.rb'
require '/data_importer/lib/importers/cna_importer.rb'
require '/data_importer/lib/importers/github_advisory_importer.rb'
require '/data_importer/lib/importers/github_user_importer.rb'
def line_sep
puts '----------' * 12
@ -28,6 +29,7 @@ def perform
import_cpes
import_cnas
import_github_advisories
import_github_usernames
end
def import_cves
@ -50,6 +52,11 @@ def import_github_advisories
GithubAdvisoryImporter.new.import
end
def import_github_usernames
line_sep
GithubUserImporter.new.import
end
def import_inthewild_cve_exploits
line_sep
InthewildCveExploitImporter.new.import

View file

@ -0,0 +1,48 @@
require '/data_importer/lib/github_api/user.rb'
class GithubUserImporter
attr_accessor :filepath, :usernames
def initialize
@filepath = '/data_importer/data/github_usernames.txt'
@usernames = File.read(filepath).split("\n")
end
def username_hashes
usernames.map do |username|
response = GithubApi::User.find(username)
response.to_h
end
end
def user_h_to_attr(user_h)
attrs = {}
attrs[:github_id] = user_h['id']
attrs[:login] = user_h['login']
attrs[:name] = user_h['name']
attrs[:avatar_url] = user_h['avatarUrl']
attrs[:bio] = user_h['bio']
attrs[:bio_html] = user_h['bioHTML']
attrs[:location] = user_h['location']
attrs
end
def bulk_insert(username_hashes)
GithubUser.bulk_insert do |worker|
username_hashes.each do |username_hash|
attrs = user_h_to_attr(username_hash)
worker.add(attrs)
end
end
end
def import
if filepath.nil?
puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line."
else
puts "Now importing GithubUsers"
usernames = username_hashes.map {|h| h[:login] }
puts "Now importing data from the following usernames: #{usernames}"
bulk_insert(username_hashes)
end
end
end