bpmcdevitt/misc_rbtools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cpe dictionary added, scans io tools module added, some work to nvd client and kenna api client

Browse source
This commit is contained in:
booboy 2019-05-19 17:51:29 -05:00
parent 1006b33c64
commit c705e0a2a8
6 changed files with 294 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
classes/kenna_api_client.rb
View file

@ -58,4 +58,4 @@ class APIClient
handle_response(response) handle_response(response)
end end
end end

3
classes/nvd_client.rb
View file

@ -3,10 +3,11 @@ require 'rest-client'
module NvdTools module NvdTools
class NvdClient class NvdClient
attr_accessor :version, :base_url, :rest_client attr_accessor :version, :base_url, :type, :rest_client
def initialize(rest_client: RestClient) def initialize(rest_client: RestClient)
@version = "1.0" @version = "1.0"
@base_url = "https://nvd.nist.gov/feeds/json/cve/#{version}/" @base_url = "https://nvd.nist.gov/feeds/json/cve/#{version}/"
@type = type
@rest_client = rest_client @rest_client = rest_client
end end

19
classes/scans_io_downloader.rb Normal file
View file

@ -0,0 +1,19 @@
require '../modules/scans_io_tools'
require 'json'
require 'restclient'
module ScansIoTools
class ScansIoDownloader
attr_accessor :base_url, :index_response
def initialize
@base_url = "https://scans.io"
@index_response = self.get_json_index.body
end
def get_json_index
response = RestClient.get "#{base_url}/json"
end
end
end

65
classes/scans_io_index_parser.rb Normal file
View file

@ -0,0 +1,65 @@
# parse the main scans.io/json file
require 'json'
module ScansIoTools
class ScansIoIndexParser
attr_accessor :json_index, :parsed_data
def initialize(json_index)
@json_index = json_index
@parsed_data = self.parse_json
end
def parse_json
JSON.parse(json_index)
end
def studies
# should be an array of 30 studies
# .keys
# ["status", "files", "long_desc", "name", "tags", "short_desc", "contact", "authors", "organization", "uniqid"]
parsed_data['studies'].map do |st|
st
end
end
def study_names
studies.map do |st|
st['name']
end
end
def study(study_name)
# pass in a study name to parse the array of all 30 studies
# .keys
# ["status", "files", "long_desc", "name", "tags", "study", "short_desc", "updated_at", "contact", "authors", "organization", "created_at", "uniqid"]
studies.select do |study|
study['name'] == "#{study_name}"
end.first
end
def file_info(study_name)
study(study_name)['files']
end
def has_filenames?(study_name)
file_lookup = file_info(study_name)
# check if the lookup includes a file in the results
if file_lookup == []
return false
else
return true
end
end
def filenames_for(study_name)
file_info(study_name).select do |file|
file['name']
end
end
def size_for(study_name)
file_info(study_name).first['size']
end
end
end

205
data/cpe/cpe-dictionary_2.3.xsd Normal file
View file

@ -0,0 +1,205 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsd:schema targetNamespace="http://cpe.mitre.org/dictionary/2.0" xmlns:cpe_dict="http://cpe.mitre.org/dictionary/2.0"
xmlns:xsd="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
xmlns:cpe-name="http://cpe.mitre.org/naming/2.0" version="2.3">
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" schemaLocation="https://www.w3.org/2009/01/xml.xsd"/>
<xsd:annotation>
<xsd:documentation xml:lang="en">This is an XML Schema for the CPE Dictionary. It is used to transfer a
collection of official CPE Names along with any necessary supporting information (title, references,
automated check, etc.) For more information, consult the CPE Specification document.</xsd:documentation>
<xsd:appinfo>
<schema>CPE Dictionary</schema>
<author>Neal Ziring, Andrew Buttner, David Waltermire</author>
<version>2.3</version>
<date>2011-07-29</date>
</xsd:appinfo>
</xsd:annotation>
<!-- =============================================================================== -->
<!-- =============================================================================== -->
<!-- =============================================================================== -->
<xsd:import namespace="http://cpe.mitre.org/naming/2.0" schemaLocation="/schema/cpe/2.3/cpe-naming_2.3.xsd"/>
<xsd:element name="cpe-list" type="cpe_dict:ListType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The cpe-list element acts as a top-level container for CPE Name items. Each
individual item must be unique. Please refer to the description of ListType for additional information
about the structure of this element.</xsd:documentation>
</xsd:annotation>
<xsd:key name="itemURIKey">
<xsd:selector xpath="./cpe_dict:cpe-item"/>
<xsd:field xpath="@name"/>
</xsd:key>
</xsd:element>
<xsd:element name="cpe-item" type="cpe_dict:ItemType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The cpe-item element denotes a single CPE Name. Please refer to the
description of ItemType for additional information about the structure of this
element.</xsd:documentation>
</xsd:annotation>
<xsd:unique name="titleLangKey">
<xsd:selector xpath="./cpe_dict:title"/>
<xsd:field xpath="@xml:lang"/>
</xsd:unique>
<xsd:unique name="notesLangKey">
<xsd:selector xpath="./cpe_dict:notes"/>
<xsd:field xpath="@xml:lang"/>
</xsd:unique>
<xsd:unique name="checkSystemKey">
<xsd:selector xpath="./cpe_dict:check"/>
<xsd:field xpath="@system"/>
</xsd:unique>
</xsd:element>
<!-- =============================================================================== -->
<!-- ============================= SUPPORTING TYPES ============================== -->
<!-- =============================================================================== -->
<xsd:complexType name="GeneratorType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The GeneratorType complex type defines an element that is used to hold
information about when a particular document was compiled, what version of the schema was used, what
tool compiled the document, and what version of that tool was used. Additional generator information is
also allowed although it is not part of the official schema. Individual organizations can place
generator information that they feel is important and it will be skipped during the validation. All that
this schema really cares about is that the stated generator information is there.</xsd:documentation>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="product_name" type="xsd:string" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation xml:lang="en">The optional product_name element specifies the name of the
application used to generate the file.</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="product_version" type="xsd:string" minOccurs="0" maxOccurs="1">
<xsd:annotation>
<xsd:documentation xml:lang="en">The optional product_version element specifies the version of the
application used to generate the file.</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="schema_version" type="xsd:decimal" minOccurs="1" maxOccurs="1">
<xsd:annotation>
<xsd:documentation xml:lang="en">The required schema_version element specifies the version of the
schema that the document has been written against and that should be used for
validation.</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:element name="timestamp" type="xsd:dateTime" minOccurs="1" maxOccurs="1">
<xsd:annotation>
<xsd:documentation xml:lang="en">The required timestamp element specifies when the particular
document was compiled. The format for the timestamp is yyyy-mm-ddThh:mm:ss. Note that the
timestamp element does not specify when an item in the document was created or modified but
rather when the actual XML document that contains the items was created. For example, a document
might pull a bunch of existing items together, each of which was created at some point in the
past. The timestamp in this case would be when this combined document was
created.</xsd:documentation>
</xsd:annotation>
</xsd:element>
<xsd:any minOccurs="0" maxOccurs="unbounded" namespace="##other" processContents="lax"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ItemType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The ItemType complex type defines an element that represents a single CPE
Name. The required name attribute is a URI which must be a unique key and should follow the URI
structure outlined in the CPE Specification. The optional title element is used to provide a
human-readable title for the platform. To support uses intended for multiple languages, this element
supports the ‘xml:lang’ attribute. At most one title element can appear for each language. The notes
element holds optional descriptive material. Multiple notes elements are allowed, but only one per
language should be used. Note that the language associated with the notes element applies to all child
note elements. The optional references element holds external info references. The optional check
element is used to call out an OVAL Definition that can confirm or reject an IT system as an instance of
the named platform. Additional elements not part of the CPE namespace are allowed and are just skipped
by validation. In essence, a dictionary file can contain additional information that a user can choose
to use or not, but this information is not required to be used or understood.</xsd:documentation>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="title" type="cpe_dict:TextType" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="notes" type="cpe_dict:NotesType" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="references" type="cpe_dict:ReferencesType" minOccurs="0" maxOccurs="1"/>
<xsd:element name="check" type="cpe_dict:CheckType" minOccurs="0" maxOccurs="unbounded"/>
<xsd:any minOccurs="0" maxOccurs="unbounded" namespace="##other" processContents="lax"/>
</xsd:sequence>
<xsd:attribute name="name" type="cpe-name:cpe22Type" use="required"/>
<xsd:attribute name="deprecated" type="xsd:boolean" use="optional" default="false"/>
<xsd:attribute name="deprecated_by" type="cpe-name:cpe22Type" use="optional"/>
<xsd:attribute name="deprecation_date" type="xsd:dateTime" use="optional"/>
</xsd:complexType>
<xsd:complexType name="ListType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The ListType complex type defines an element that is used to hold a
collection of individual items. The required generator section provides information about when the
definition file was compiled and under what version. Additional elements not part of the CPE namespace
are allowed and are just skipped by validation. In essence, a dictionary file can contain additional
information that a user can choose to use or not, but this information is not required to be used or
understood.</xsd:documentation>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="generator" type="cpe_dict:GeneratorType" minOccurs="0" maxOccurs="1"/>
<xsd:element ref="cpe_dict:cpe-item" minOccurs="1" maxOccurs="unbounded"/>
<xsd:any minOccurs="0" maxOccurs="unbounded" namespace="##other" processContents="lax"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="TextType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The TextType complex type allows the xml:lang attribute to associate a
specific language with an element's string content.</xsd:documentation>
</xsd:annotation>
<xsd:simpleContent>
<xsd:extension base="xsd:string">
<xsd:attribute ref="xml:lang"/>
</xsd:extension>
</xsd:simpleContent>
</xsd:complexType>
<xsd:complexType name="NotesType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The NotesType complex type defines an element that consists of one or more
child note elements. It is assumed that each of these note elements is representative of the same
language as defined by their parent.</xsd:documentation>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="note" type="xsd:string" minOccurs="1" maxOccurs="unbounded"/>
</xsd:sequence>
<xsd:attribute ref="xml:lang"/>
</xsd:complexType>
<xsd:complexType name="ReferencesType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The ReferencesType complex type defines an element used to hold a
collection of individual references. Each reference consists of a piece of text (intended to be
human-readable) and a URI (intended to be a URL, and point to a real resource) and is used to point to
extra descriptive material, for example a supplier's web site or platform
documentation.</xsd:documentation>
</xsd:annotation>
<xsd:sequence>
<xsd:element name="reference" minOccurs="1" maxOccurs="unbounded">
<xsd:complexType>
<xsd:simpleContent>
<xsd:extension base="xsd:string">
<xsd:attribute name="href" type="xsd:anyURI"/>
</xsd:extension>
</xsd:simpleContent>
</xsd:complexType>
</xsd:element>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="CheckType">
<xsd:annotation>
<xsd:documentation xml:lang="en">The CheckType complex type is used to define an element to hold information
about an individual check. It includes a checking system specification URI, string content, and an
optional external file reference. The checking system specification should be the URI for a particular
version of OVAL or a related system testing language, and the content will be an identifier of a test
written in that language. The external file reference could be used to point to the file in which the
content test identifier is defined.</xsd:documentation>
</xsd:annotation>
<xsd:simpleContent>
<xsd:extension base="xsd:string">
<xsd:attribute name="system" type="xsd:anyURI" use="required"/>
<xsd:attribute name="href" type="xsd:anyURI" use="optional"/>
</xsd:extension>
</xsd:simpleContent>
</xsd:complexType>
<!-- ================================================== -->
<!-- ===== Change History -->
<!-- ================================================== -->
<!--
v2.3 - Made title to minOccurs = 0. Removed namePattern and referencing CPE Naming schema.
-->
</xsd:schema>

2
modules/scans_io_tools.rb Normal file
View file

@ -0,0 +1,2 @@
module ScansIoTools
end