From c705e0a2a86a92b383e1d1604e207cdf8eae1174 Mon Sep 17 00:00:00 2001 From: booboy Date: Sun, 19 May 2019 17:51:29 -0500 Subject: [PATCH] cpe dictionary added, scans io tools module added, some work to nvd client and kenna api client --- classes/kenna_api_client.rb | 2 +- classes/nvd_client.rb | 3 +- classes/scans_io_downloader.rb | 19 +++ classes/scans_io_index_parser.rb | 65 ++++++++++ data/cpe/cpe-dictionary_2.3.xsd | 205 +++++++++++++++++++++++++++++++ modules/scans_io_tools.rb | 2 + 6 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 classes/scans_io_downloader.rb create mode 100644 classes/scans_io_index_parser.rb create mode 100644 data/cpe/cpe-dictionary_2.3.xsd create mode 100644 modules/scans_io_tools.rb diff --git a/classes/kenna_api_client.rb b/classes/kenna_api_client.rb index 2752c6f..0ee2e70 100644 --- a/classes/kenna_api_client.rb +++ b/classes/kenna_api_client.rb @@ -58,4 +58,4 @@ class APIClient handle_response(response) end -end +end \ No newline at end of file diff --git a/classes/nvd_client.rb b/classes/nvd_client.rb index 3316749..c58c57a 100644 --- a/classes/nvd_client.rb +++ b/classes/nvd_client.rb @@ -3,10 +3,11 @@ require 'rest-client' module NvdTools class NvdClient - attr_accessor :version, :base_url, :rest_client + attr_accessor :version, :base_url, :type, :rest_client def initialize(rest_client: RestClient) @version = "1.0" @base_url = "https://nvd.nist.gov/feeds/json/cve/#{version}/" + @type = type @rest_client = rest_client end diff --git a/classes/scans_io_downloader.rb b/classes/scans_io_downloader.rb new file mode 100644 index 0000000..a17131d --- /dev/null +++ b/classes/scans_io_downloader.rb @@ -0,0 +1,19 @@ +require '../modules/scans_io_tools' +require 'json' +require 'restclient' + +module ScansIoTools + class ScansIoDownloader + attr_accessor :base_url, :index_response + + def initialize + @base_url = "https://scans.io" + @index_response = self.get_json_index.body + end + + def get_json_index + response = RestClient.get "#{base_url}/json" + end + + end +end \ No newline at end of file diff --git a/classes/scans_io_index_parser.rb b/classes/scans_io_index_parser.rb new file mode 100644 index 0000000..925b484 --- /dev/null +++ b/classes/scans_io_index_parser.rb @@ -0,0 +1,65 @@ +# parse the main scans.io/json file +require 'json' + +module ScansIoTools + class ScansIoIndexParser + attr_accessor :json_index, :parsed_data + + def initialize(json_index) + @json_index = json_index + @parsed_data = self.parse_json + end + + def parse_json + JSON.parse(json_index) + end + + def studies + # should be an array of 30 studies + # .keys + # ["status", "files", "long_desc", "name", "tags", "short_desc", "contact", "authors", "organization", "uniqid"] + parsed_data['studies'].map do |st| + st + end + end + + def study_names + studies.map do |st| + st['name'] + end + end + + def study(study_name) + # pass in a study name to parse the array of all 30 studies + # .keys + # ["status", "files", "long_desc", "name", "tags", "study", "short_desc", "updated_at", "contact", "authors", "organization", "created_at", "uniqid"] + studies.select do |study| + study['name'] == "#{study_name}" + end.first + end + + def file_info(study_name) + study(study_name)['files'] + end + + def has_filenames?(study_name) + file_lookup = file_info(study_name) + # check if the lookup includes a file in the results + if file_lookup == [] + return false + else + return true + end + end + + def filenames_for(study_name) + file_info(study_name).select do |file| + file['name'] + end + end + + def size_for(study_name) + file_info(study_name).first['size'] + end + end +end \ No newline at end of file diff --git a/data/cpe/cpe-dictionary_2.3.xsd b/data/cpe/cpe-dictionary_2.3.xsd new file mode 100644 index 0000000..304e095 --- /dev/null +++ b/data/cpe/cpe-dictionary_2.3.xsd @@ -0,0 +1,205 @@ + + + + + This is an XML Schema for the CPE Dictionary. It is used to transfer a + collection of official CPE Names along with any necessary supporting information (title, references, + automated check, etc.) For more information, consult the CPE Specification document. + + CPE Dictionary + Neal Ziring, Andrew Buttner, David Waltermire + 2.3 + 2011-07-29 + + + + + + + + + The cpe-list element acts as a top-level container for CPE Name items. Each + individual item must be unique. Please refer to the description of ListType for additional information + about the structure of this element. + + + + + + + + + The cpe-item element denotes a single CPE Name. Please refer to the + description of ItemType for additional information about the structure of this + element. + + + + + + + + + + + + + + + + + + + + The GeneratorType complex type defines an element that is used to hold + information about when a particular document was compiled, what version of the schema was used, what + tool compiled the document, and what version of that tool was used. Additional generator information is + also allowed although it is not part of the official schema. Individual organizations can place + generator information that they feel is important and it will be skipped during the validation. All that + this schema really cares about is that the stated generator information is there. + + + + + The optional product_name element specifies the name of the + application used to generate the file. + + + + + The optional product_version element specifies the version of the + application used to generate the file. + + + + + The required schema_version element specifies the version of the + schema that the document has been written against and that should be used for + validation. + + + + + The required timestamp element specifies when the particular + document was compiled. The format for the timestamp is yyyy-mm-ddThh:mm:ss. Note that the + timestamp element does not specify when an item in the document was created or modified but + rather when the actual XML document that contains the items was created. For example, a document + might pull a bunch of existing items together, each of which was created at some point in the + past. The timestamp in this case would be when this combined document was + created. + + + + + + + + The ItemType complex type defines an element that represents a single CPE + Name. The required name attribute is a URI which must be a unique key and should follow the URI + structure outlined in the CPE Specification. The optional title element is used to provide a + human-readable title for the platform. To support uses intended for multiple languages, this element + supports the ‘xml:lang’ attribute. At most one title element can appear for each language. The notes + element holds optional descriptive material. Multiple notes elements are allowed, but only one per + language should be used. Note that the language associated with the notes element applies to all child + note elements. The optional references element holds external info references. The optional check + element is used to call out an OVAL Definition that can confirm or reject an IT system as an instance of + the named platform. Additional elements not part of the CPE namespace are allowed and are just skipped + by validation. In essence, a dictionary file can contain additional information that a user can choose + to use or not, but this information is not required to be used or understood. + + + + + + + + + + + + + + + + The ListType complex type defines an element that is used to hold a + collection of individual items. The required generator section provides information about when the + definition file was compiled and under what version. Additional elements not part of the CPE namespace + are allowed and are just skipped by validation. In essence, a dictionary file can contain additional + information that a user can choose to use or not, but this information is not required to be used or + understood. + + + + + + + + + + The TextType complex type allows the xml:lang attribute to associate a + specific language with an element's string content. + + + + + + + + + + The NotesType complex type defines an element that consists of one or more + child note elements. It is assumed that each of these note elements is representative of the same + language as defined by their parent. + + + + + + + + + The ReferencesType complex type defines an element used to hold a + collection of individual references. Each reference consists of a piece of text (intended to be + human-readable) and a URI (intended to be a URL, and point to a real resource) and is used to point to + extra descriptive material, for example a supplier's web site or platform + documentation. + + + + + + + + + + + + + + + + The CheckType complex type is used to define an element to hold information + about an individual check. It includes a checking system specification URI, string content, and an + optional external file reference. The checking system specification should be the URI for a particular + version of OVAL or a related system testing language, and the content will be an identifier of a test + written in that language. The external file reference could be used to point to the file in which the + content test identifier is defined. + + + + + + + + + + + + + + + diff --git a/modules/scans_io_tools.rb b/modules/scans_io_tools.rb new file mode 100644 index 0000000..3779a52 --- /dev/null +++ b/modules/scans_io_tools.rb @@ -0,0 +1,2 @@ +module ScansIoTools +end \ No newline at end of file