do a github api repository owner lookup and combine that api response with our github user lookup and put it all in GithubUser
This commit is contained in:
parent
2dfd9981f7
commit
8ce7d5257c
4 changed files with 78 additions and 3 deletions
|
@ -8,6 +8,7 @@ class CreateGithubUsers < ActiveRecord::Migration[7.0]
|
||||||
t.string :bio
|
t.string :bio
|
||||||
t.text :bio_html
|
t.text :bio_html
|
||||||
t.string :location
|
t.string :location
|
||||||
|
t.jsonb :repositories
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -107,6 +107,7 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
|
||||||
t.string "bio"
|
t.string "bio"
|
||||||
t.text "bio_html"
|
t.text "bio_html"
|
||||||
t.string "location"
|
t.string "location"
|
||||||
|
t.jsonb "repositories"
|
||||||
end
|
end
|
||||||
|
|
||||||
create_table "inthewild_cve_exploits", force: :cascade do |t|
|
create_table "inthewild_cve_exploits", force: :cascade do |t|
|
||||||
|
|
69
lib/github_api/owner_repos.rb
Normal file
69
lib/github_api/owner_repos.rb
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
require '/data_importer/lib/github_api/github_api.rb'
|
||||||
|
|
||||||
|
module GithubApi
|
||||||
|
class OwnerRepos
|
||||||
|
OwnerReposQuery = GithubApi::Client.parse <<-'GRAPHQL'
|
||||||
|
query($owner: String!, $cursor: String) {
|
||||||
|
repositoryOwner(login: $owner) {
|
||||||
|
repositories(
|
||||||
|
first: 100
|
||||||
|
ownerAffiliations: OWNER
|
||||||
|
privacy: PUBLIC
|
||||||
|
isFork: false
|
||||||
|
isLocked: false
|
||||||
|
orderBy: {field: NAME, direction: ASC}
|
||||||
|
after: $cursor
|
||||||
|
) {
|
||||||
|
totalCount
|
||||||
|
pageInfo {
|
||||||
|
hasNextPage
|
||||||
|
endCursor
|
||||||
|
}
|
||||||
|
nodes {
|
||||||
|
name
|
||||||
|
url
|
||||||
|
homepageUrl
|
||||||
|
mirrorUrl
|
||||||
|
openGraphImageUrl
|
||||||
|
resourcePath
|
||||||
|
description
|
||||||
|
descriptionHTML
|
||||||
|
shortDescriptionHTML
|
||||||
|
createdAt
|
||||||
|
pushedAt
|
||||||
|
updatedAt
|
||||||
|
forkCount
|
||||||
|
hasIssuesEnabled
|
||||||
|
hasProjectsEnabled
|
||||||
|
hasWikiEnabled
|
||||||
|
isArchived
|
||||||
|
isFork
|
||||||
|
isInOrganization
|
||||||
|
isLocked
|
||||||
|
isMirror
|
||||||
|
isPrivate
|
||||||
|
isTemplate
|
||||||
|
licenseInfo {
|
||||||
|
id
|
||||||
|
}
|
||||||
|
lockReason
|
||||||
|
usesCustomOpenGraphImage
|
||||||
|
visibility
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
GRAPHQL
|
||||||
|
|
||||||
|
def self.find(username)
|
||||||
|
response = GithubApi::Client.query(OwnerReposQuery, variables: { owner: username })
|
||||||
|
if response.errors.any?
|
||||||
|
raise QueryExecutionError.new(response.errors[:data].join(", "))
|
||||||
|
else
|
||||||
|
response.data.repository_owner.repositories.nodes.map(&:to_h)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
class QueryExecutionError < StandardError; end
|
|
@ -1,4 +1,5 @@
|
||||||
require '/data_importer/lib/github_api/user.rb'
|
require '/data_importer/lib/github_api/user.rb'
|
||||||
|
require '/data_importer/lib/github_api/owner_repos.rb'
|
||||||
|
|
||||||
class GithubUserImporter
|
class GithubUserImporter
|
||||||
attr_accessor :filepath, :usernames
|
attr_accessor :filepath, :usernames
|
||||||
|
@ -9,8 +10,10 @@ class GithubUserImporter
|
||||||
|
|
||||||
def username_hashes
|
def username_hashes
|
||||||
usernames.map do |username|
|
usernames.map do |username|
|
||||||
response = GithubApi::User.find(username)
|
username_response = GithubApi::User.find(username)
|
||||||
response.to_h
|
username_repos = GithubApi::OwnerRepos.find(username)
|
||||||
|
repos_hash = { 'repositories' => username_repos }
|
||||||
|
username_response.to_h.merge(repos_hash)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -23,6 +26,7 @@ class GithubUserImporter
|
||||||
attrs[:bio] = user_h['bio']
|
attrs[:bio] = user_h['bio']
|
||||||
attrs[:bio_html] = user_h['bioHTML']
|
attrs[:bio_html] = user_h['bioHTML']
|
||||||
attrs[:location] = user_h['location']
|
attrs[:location] = user_h['location']
|
||||||
|
attrs[:repositories] = user_h['repositories']
|
||||||
attrs
|
attrs
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -40,7 +44,7 @@ class GithubUserImporter
|
||||||
puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line."
|
puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line."
|
||||||
else
|
else
|
||||||
puts "Now importing GithubUsers"
|
puts "Now importing GithubUsers"
|
||||||
usernames = username_hashes.map {|h| h[:login] }
|
usernames = username_hashes.map {|h| h['login' }
|
||||||
puts "Now importing data from the following usernames: #{usernames}"
|
puts "Now importing data from the following usernames: #{usernames}"
|
||||||
bulk_insert(username_hashes)
|
bulk_insert(username_hashes)
|
||||||
end
|
end
|
||||||
|
|
Loading…
Add table
Reference in a new issue