if you create a text file at ./data/github_usernames.txt with one username per line GithubUserImporter will use this data to import a GithubUser for each one
This commit is contained in:
parent
db6afbfd59
commit
d26395def2
15 changed files with 93 additions and 1 deletions
|
@ -62,6 +62,14 @@ For now unauthenticated api over localhost:3000 until I put in some basic token
|
|||
get "/github_advisories/:ghsa_id", to: "github_advisories#show"
|
||||
```
|
||||
|
||||
#### GithubUsers
|
||||
Create a text file named `./data/github_usernames.txt` with one username per line
|
||||
There is a seed task that will read this file and perform an API call to github API and store the data in DB for each user.
|
||||
```
|
||||
get "/github_users", to: "github_users#index"
|
||||
get "/github_users/:username", to: "github_users#show"
|
||||
```
|
||||
|
||||
#### GithubPocs
|
||||
```
|
||||
get "/github_pocs", to: "github_pocs#index"
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
class CnasController < ApplicationController
|
||||
def index
|
||||
@cnas = Cna.all
|
||||
render json: @cnas.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
class CpesController < ApplicationController
|
||||
def index
|
||||
@cpes = Cpe.all
|
||||
render json: @cpes.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
class CvemonCvesController < ApplicationController
|
||||
def index
|
||||
@pocs = CvemonCve.all
|
||||
render json: @pocs.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
class CvesController < ApplicationController
|
||||
def index
|
||||
@cves = Cve.all
|
||||
render json: @cves.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
class GithubAdvisoriesController < ApplicationController
|
||||
def index
|
||||
@advisories = GithubAdvisory.all
|
||||
render json: @advisories.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
@advisory = GithubAdivsory.find_by_ghsa_id(params[:ghsa_id])
|
||||
@advisory = GithubAdvisory.find_by_ghsa_id(params[:ghsa_id])
|
||||
render json: @advisory.to_json
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
class GithubPocsController < ApplicationController
|
||||
def index
|
||||
@pocs = GithubPoc.all
|
||||
render json: @pocs.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
11
app/controllers/github_users_controller.rb
Normal file
11
app/controllers/github_users_controller.rb
Normal file
|
@ -0,0 +1,11 @@
|
|||
class GithubUsersController < ApplicationController
|
||||
def index
|
||||
@users = GithubUser.all
|
||||
render json: @users.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
@user = GithubUser.find_by_username(params[:username])
|
||||
render json: @user.to_json
|
||||
end
|
||||
end
|
|
@ -1,6 +1,7 @@
|
|||
class TrickestPocCvesController < ApplicationController
|
||||
def index
|
||||
@pocs = TrickestPocCve.all
|
||||
render json: @pocs.to_json
|
||||
end
|
||||
|
||||
def show
|
||||
|
|
5
app/models/github_user.rb
Normal file
5
app/models/github_user.rb
Normal file
|
@ -0,0 +1,5 @@
|
|||
class GithubUser< ActiveRecord::Base
|
||||
def self.find_by_username(username)
|
||||
find_by(login: username)
|
||||
end
|
||||
end
|
1
app/views/github_users/index.html.erb
Normal file
1
app/views/github_users/index.html.erb
Normal file
|
@ -0,0 +1 @@
|
|||
<h1>Cpes#index</h1>
|
2
app/views/github_users/show.html.erb
Normal file
2
app/views/github_users/show.html.erb
Normal file
|
@ -0,0 +1,2 @@
|
|||
@users
|
||||
|
|
@ -33,4 +33,7 @@ Rails.application.routes.draw do
|
|||
get "/github_advisories", to: "github_advisories#index"
|
||||
get "/github_advisories/:ghsa_id", to: "github_advisories#show"
|
||||
|
||||
get "/github_users", to: "github_users#index"
|
||||
get "/github_users/:username", to: "github_users#show"
|
||||
|
||||
end
|
||||
|
|
|
@ -14,6 +14,7 @@ require '/data_importer/lib/importers/trickest_poc_cve_importer.rb'
|
|||
require '/data_importer/lib/importers/cvemon_cve_importer.rb'
|
||||
require '/data_importer/lib/importers/cna_importer.rb'
|
||||
require '/data_importer/lib/importers/github_advisory_importer.rb'
|
||||
require '/data_importer/lib/importers/github_user_importer.rb'
|
||||
|
||||
def line_sep
|
||||
puts '----------' * 12
|
||||
|
@ -28,6 +29,7 @@ def perform
|
|||
import_cpes
|
||||
import_cnas
|
||||
import_github_advisories
|
||||
import_github_usernames
|
||||
end
|
||||
|
||||
def import_cves
|
||||
|
@ -50,6 +52,11 @@ def import_github_advisories
|
|||
GithubAdvisoryImporter.new.import
|
||||
end
|
||||
|
||||
def import_github_usernames
|
||||
line_sep
|
||||
GithubUserImporter.new.import
|
||||
end
|
||||
|
||||
def import_inthewild_cve_exploits
|
||||
line_sep
|
||||
InthewildCveExploitImporter.new.import
|
||||
|
|
48
lib/importers/github_user_importer.rb
Normal file
48
lib/importers/github_user_importer.rb
Normal file
|
@ -0,0 +1,48 @@
|
|||
require '/data_importer/lib/github_api/user.rb'
|
||||
|
||||
class GithubUserImporter
|
||||
attr_accessor :filepath, :usernames
|
||||
def initialize
|
||||
@filepath = '/data_importer/data/github_usernames.txt'
|
||||
@usernames = File.read(filepath).split("\n")
|
||||
end
|
||||
|
||||
def username_hashes
|
||||
usernames.map do |username|
|
||||
response = GithubApi::User.find(username)
|
||||
response.to_h
|
||||
end
|
||||
end
|
||||
|
||||
def user_h_to_attr(user_h)
|
||||
attrs = {}
|
||||
attrs[:github_id] = user_h['id']
|
||||
attrs[:login] = user_h['login']
|
||||
attrs[:name] = user_h['name']
|
||||
attrs[:avatar_url] = user_h['avatarUrl']
|
||||
attrs[:bio] = user_h['bio']
|
||||
attrs[:bio_html] = user_h['bioHTML']
|
||||
attrs[:location] = user_h['location']
|
||||
attrs
|
||||
end
|
||||
|
||||
def bulk_insert(username_hashes)
|
||||
GithubUser.bulk_insert do |worker|
|
||||
username_hashes.each do |username_hash|
|
||||
attrs = user_h_to_attr(username_hash)
|
||||
worker.add(attrs)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import
|
||||
if filepath.nil?
|
||||
puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line."
|
||||
else
|
||||
puts "Now importing GithubUsers"
|
||||
usernames = username_hashes.map {|h| h[:login] }
|
||||
puts "Now importing data from the following usernames: #{usernames}"
|
||||
bulk_insert(username_hashes)
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Add table
Reference in a new issue