made a json helper to help me snake case / symbolize keys for prep to upsert_all into db

This commit is contained in:
Brendan McDevitt 2022-04-18 20:01:57 -05:00
parent 6f02a95a4a
commit 9fd17c606a
2 changed files with 19 additions and 29 deletions

View file

@ -1,6 +1,6 @@
require 'json' require 'json'
require '/data_importer/lib/json_helper.rb'
require 'rest-client' require 'rest-client'
require 'bulk_insert'
class CnaImporter class CnaImporter
attr_accessor :url attr_accessor :url
@ -16,46 +16,23 @@ class CnaImporter
end end
def parse_res(response) def parse_res(response)
JSON.parse(response.body) JSON.parse(response.body, symbolize_names: true)
end end
def get_json def get_json
res = send_request_rest res = send_request_rest
if res.code == 200 if res.code == 200
parse_res(res) json = parse_res(res)
JsonHelper.deep_transform_keys(json)
else else
"HTTP Status: #{res.code}" "HTTP Status: #{res.code}"
end end
end end
def json_to_hash(json)
data_hash = {}
data_hash[:short_name] = json['shortName']
data_hash[:cna_id] = json['cnaID']
data_hash[:organization_name] = json['organizationName']
data_hash[:scope] = json['scope']
data_hash[:contact] = json['contact']
data_hash[:disclosure_policy] = json['disclosurePolicy']
data_hash[:security_advisories] = json['securityAdvisories']
data_hash[:resources] = json['resources']
data_hash[:cna] = json['CNA']
data_hash[:country] = json['country']
data_hash
end
def bulk_insert(cves)
Cna.bulk_insert do |worker|
cves.each do |attrs|
worker.add(attrs)
end
end
end
def import def import
json = get_json jsons = get_json
attrs = json.map {|j| json_to_hash(j) }
puts "Now importing CNAs." puts "Now importing CNAs."
bulk_insert(attrs) Cna.upsert_all(jsons, unique_by: :cna_id)
end end
end end

13
lib/json_helper.rb Normal file
View file

@ -0,0 +1,13 @@
class JsonHelper
def self.deep_transform_keys(json_hash)
if json_hash.is_a? Array
json_hash.map {|jh| symbolize_names_snake_case(jh) }
else
symbolize_names_snake_case(json_hash)
end
end
def self.symbolize_names_snake_case(json_hash)
json_hash.deep_transform_keys {|k| k.to_s.underscore.to_sym }
end
end