From 8ce7d5257cfa77758d6dfeac0d8e6c7ec0fc9661 Mon Sep 17 00:00:00 2001 From: Brendan McDevitt Date: Tue, 12 Apr 2022 12:24:50 -0500 Subject: [PATCH] do a github api repository owner lookup and combine that api response with our github user lookup and put it all in GithubUser --- .../20220411174826_create_github_users.rb | 1 + db/schema.rb | 1 + lib/github_api/owner_repos.rb | 69 +++++++++++++++++++ lib/importers/github_user_importer.rb | 10 ++- 4 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 lib/github_api/owner_repos.rb diff --git a/db/migrate/20220411174826_create_github_users.rb b/db/migrate/20220411174826_create_github_users.rb index 42d8368..ee9a10b 100644 --- a/db/migrate/20220411174826_create_github_users.rb +++ b/db/migrate/20220411174826_create_github_users.rb @@ -8,6 +8,7 @@ class CreateGithubUsers < ActiveRecord::Migration[7.0] t.string :bio t.text :bio_html t.string :location + t.jsonb :repositories end end end diff --git a/db/schema.rb b/db/schema.rb index f808210..a6ae81c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -107,6 +107,7 @@ ActiveRecord::Schema[7.0].define(version: 2022_04_11_181501) do t.string "bio" t.text "bio_html" t.string "location" + t.jsonb "repositories" end create_table "inthewild_cve_exploits", force: :cascade do |t| diff --git a/lib/github_api/owner_repos.rb b/lib/github_api/owner_repos.rb new file mode 100644 index 0000000..f284a4d --- /dev/null +++ b/lib/github_api/owner_repos.rb @@ -0,0 +1,69 @@ +require '/data_importer/lib/github_api/github_api.rb' + +module GithubApi + class OwnerRepos + OwnerReposQuery = GithubApi::Client.parse <<-'GRAPHQL' + query($owner: String!, $cursor: String) { + repositoryOwner(login: $owner) { + repositories( + first: 100 + ownerAffiliations: OWNER + privacy: PUBLIC + isFork: false + isLocked: false + orderBy: {field: NAME, direction: ASC} + after: $cursor + ) { + totalCount + pageInfo { + hasNextPage + endCursor + } + nodes { + name + url + homepageUrl + mirrorUrl + openGraphImageUrl + resourcePath + description + descriptionHTML + shortDescriptionHTML + createdAt + pushedAt + updatedAt + forkCount + hasIssuesEnabled + hasProjectsEnabled + hasWikiEnabled + isArchived + isFork + isInOrganization + isLocked + isMirror + isPrivate + isTemplate + licenseInfo { + id + } + lockReason + usesCustomOpenGraphImage + visibility + } + } + } + } + GRAPHQL + + def self.find(username) + response = GithubApi::Client.query(OwnerReposQuery, variables: { owner: username }) + if response.errors.any? + raise QueryExecutionError.new(response.errors[:data].join(", ")) + else + response.data.repository_owner.repositories.nodes.map(&:to_h) + end + end + end +end + +class QueryExecutionError < StandardError; end \ No newline at end of file diff --git a/lib/importers/github_user_importer.rb b/lib/importers/github_user_importer.rb index d19170c..43ca492 100644 --- a/lib/importers/github_user_importer.rb +++ b/lib/importers/github_user_importer.rb @@ -1,4 +1,5 @@ require '/data_importer/lib/github_api/user.rb' +require '/data_importer/lib/github_api/owner_repos.rb' class GithubUserImporter attr_accessor :filepath, :usernames @@ -9,8 +10,10 @@ class GithubUserImporter def username_hashes usernames.map do |username| - response = GithubApi::User.find(username) - response.to_h + username_response = GithubApi::User.find(username) + username_repos = GithubApi::OwnerRepos.find(username) + repos_hash = { 'repositories' => username_repos } + username_response.to_h.merge(repos_hash) end end @@ -23,6 +26,7 @@ class GithubUserImporter attrs[:bio] = user_h['bio'] attrs[:bio_html] = user_h['bioHTML'] attrs[:location] = user_h['location'] + attrs[:repositories] = user_h['repositories'] attrs end @@ -40,7 +44,7 @@ class GithubUserImporter puts "Please provide a filepath in the projects data dir named github_usernames.txt with one username per line." else puts "Now importing GithubUsers" - usernames = username_hashes.map {|h| h[:login] } + usernames = username_hashes.map {|h| h['login' } puts "Now importing data from the following usernames: #{usernames}" bulk_insert(username_hashes) end