made a json helper to help me snake case / symbolize keys for prep to upsert_all into db
This commit is contained in:
parent
6f02a95a4a
commit
9fd17c606a
2 changed files with 19 additions and 29 deletions
|
@ -1,6 +1,6 @@
|
|||
require 'json'
|
||||
require '/data_importer/lib/json_helper.rb'
|
||||
require 'rest-client'
|
||||
require 'bulk_insert'
|
||||
|
||||
class CnaImporter
|
||||
attr_accessor :url
|
||||
|
@ -16,46 +16,23 @@ class CnaImporter
|
|||
end
|
||||
|
||||
def parse_res(response)
|
||||
JSON.parse(response.body)
|
||||
JSON.parse(response.body, symbolize_names: true)
|
||||
end
|
||||
|
||||
def get_json
|
||||
res = send_request_rest
|
||||
if res.code == 200
|
||||
parse_res(res)
|
||||
json = parse_res(res)
|
||||
JsonHelper.deep_transform_keys(json)
|
||||
else
|
||||
"HTTP Status: #{res.code}"
|
||||
end
|
||||
end
|
||||
|
||||
def json_to_hash(json)
|
||||
data_hash = {}
|
||||
data_hash[:short_name] = json['shortName']
|
||||
data_hash[:cna_id] = json['cnaID']
|
||||
data_hash[:organization_name] = json['organizationName']
|
||||
data_hash[:scope] = json['scope']
|
||||
data_hash[:contact] = json['contact']
|
||||
data_hash[:disclosure_policy] = json['disclosurePolicy']
|
||||
data_hash[:security_advisories] = json['securityAdvisories']
|
||||
data_hash[:resources] = json['resources']
|
||||
data_hash[:cna] = json['CNA']
|
||||
data_hash[:country] = json['country']
|
||||
data_hash
|
||||
end
|
||||
|
||||
def bulk_insert(cves)
|
||||
Cna.bulk_insert do |worker|
|
||||
cves.each do |attrs|
|
||||
worker.add(attrs)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import
|
||||
json = get_json
|
||||
attrs = json.map {|j| json_to_hash(j) }
|
||||
jsons = get_json
|
||||
puts "Now importing CNAs."
|
||||
bulk_insert(attrs)
|
||||
Cna.upsert_all(jsons, unique_by: :cna_id)
|
||||
end
|
||||
|
||||
end
|
||||
|
|
13
lib/json_helper.rb
Normal file
13
lib/json_helper.rb
Normal file
|
@ -0,0 +1,13 @@
|
|||
class JsonHelper
|
||||
def self.deep_transform_keys(json_hash)
|
||||
if json_hash.is_a? Array
|
||||
json_hash.map {|jh| symbolize_names_snake_case(jh) }
|
||||
else
|
||||
symbolize_names_snake_case(json_hash)
|
||||
end
|
||||
end
|
||||
|
||||
def self.symbolize_names_snake_case(json_hash)
|
||||
json_hash.deep_transform_keys {|k| k.to_s.underscore.to_sym }
|
||||
end
|
||||
end
|
Loading…
Add table
Reference in a new issue