do a github api repository owner lookup and combine that api response with our github user lookup and put it all in GithubUser

This commit is contained in:
Brendan McDevitt 2022-04-12 12:24:50 -05:00
parent 2dfd9981f7
commit 8ce7d5257c
4 changed files with 78 additions and 3 deletions

View file

@ -8,6 +8,7 @@ class CreateGithubUsers < ActiveRecord::Migration[7.0]
t.string :bio t.string :bio
t.text :bio_html t.text :bio_html
t.string :location t.string :location
t.jsonb :repositories
end end
end end
end end

View file

@ -107,6 +107,7 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do
t.string "bio" t.string "bio"
t.text "bio_html" t.text "bio_html"
t.string "location" t.string "location"
t.jsonb "repositories"
end end
create_table "inthewild_cve_exploits", force: :cascade do |t| create_table "inthewild_cve_exploits", force: :cascade do |t|

View file

@ -0,0 +1,69 @@
require '/data_importer/lib/github_api/github_api.rb'
module GithubApi
class OwnerRepos
OwnerReposQuery = GithubApi::Client.parse <<-'GRAPHQL'
query($owner: String!, $cursor: String) {
repositoryOwner(login: $owner) {
repositories(
first: 100
ownerAffiliations: OWNER
privacy: PUBLIC
isFork: false
isLocked: false
orderBy: {field: NAME, direction: ASC}
after: $cursor
) {
totalCount
pageInfo {
hasNextPage
endCursor
}
nodes {
name
url
homepageUrl
mirrorUrl
openGraphImageUrl
resourcePath
description
descriptionHTML
shortDescriptionHTML
createdAt
pushedAt
updatedAt
forkCount
hasIssuesEnabled
hasProjectsEnabled
hasWikiEnabled
isArchived
isFork
isInOrganization
isLocked
isMirror
isPrivate
isTemplate
licenseInfo {
id
}
lockReason
usesCustomOpenGraphImage
visibility
}
}
}
}
GRAPHQL
def self.find(username)
response = GithubApi::Client.query(OwnerReposQuery, variables: { owner: username })
if response.errors.any?
raise QueryExecutionError.new(response.errors[:data].join(", "))
else
response.data.repository_owner.repositories.nodes.map(&:to_h)
end
end
end
end
class QueryExecutionError < StandardError; end

View file

@ -1,4 +1,5 @@
require '/data_importer/lib/github_api/user.rb' require '/data_importer/lib/github_api/user.rb'
require '/data_importer/lib/github_api/owner_repos.rb'
class GithubUserImporter class GithubUserImporter
attr_accessor :filepath, :usernames attr_accessor :filepath, :usernames
@ -9,8 +10,10 @@ class GithubUserImporter
def username_hashes def username_hashes
usernames.map do |username| usernames.map do |username|
response = GithubApi::User.find(username) username_response = GithubApi::User.find(username)
response.to_h username_repos = GithubApi::OwnerRepos.find(username)
repos_hash = { 'repositories' => username_repos }
username_response.to_h.merge(repos_hash)
end end
end end
@ -23,6 +26,7 @@ class GithubUserImporter
attrs[:bio] = user_h['bio'] attrs[:bio] = user_h['bio']
attrs[:bio_html] = user_h['bioHTML'] attrs[:bio_html] = user_h['bioHTML']
attrs[:location] = user_h['location'] attrs[:location] = user_h['location']
attrs[:repositories] = user_h['repositories']
attrs attrs
end end
@ -40,7 +44,7 @@ class GithubUserImporter
puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line." puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line."
else else
puts "Now importing GithubUsers" puts "Now importing GithubUsers"
usernames = username_hashes.map {|h| h[:login] } usernames = username_hashes.map {|h| h['login' }
puts "Now importing data from the following usernames: #{usernames}" puts "Now importing data from the following usernames: #{usernames}"
bulk_insert(username_hashes) bulk_insert(username_hashes)
end end