add command-line wrapper

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "tools/cvelist"]
 	path = tools/cvelist
 	url = https://github.com/CVEProject/cvelist
+[submodule "containers/domain_registration_tracker/nicinfo"]
+	path = containers/domain_registration_tracker/nicinfo
+	url = git@github.com:arineng/nicinfo.git

containers/domain_registration_tracker/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpinelab/ruby-dev
+
+RUN apt-get update \
+ && apt-get install --assume-yes --no-install-recommends --no-install-suggests \
+ pry \
+ && apt-get install --assume-yes --no-install-recommends --no-install-suggests\
+ jq \
+ && rm -rf /var/lib/apt/lists/* \
+ && wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
+    chmod +x /usr/bin/yq \
+&& curl -sSL https://bit.ly/install-xq | bash 

containers/domain_registration_tracker/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+volumes:
+  bundle:       { driver: local }
+  node_modules: { driver: local }
+  config:       { driver: local }
+services:
+  app:
+    build: .
+    ports: ["5000:5000"]
+    volumes:
+      - .:/app
+      - bundle:/bundle
+      - node_modules:/app/node_modules
+      - config:/config

containers/domain_registration_tracker/nicinfo

@@ -0,0 +1 @@
+Subproject commit ac8b5d1b36ce7759d9e396af5677a1b4174f0650

containers/domain_registration_tracker/src/get_iana_registrar_ids.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# This script will download the list of domain registrars from iana.org
+#
+wget -O data/registrar-ids.xml https://www.iana.org/assignments/registrar-ids/registrar-ids.xml
+wget -O data/rdap_tech_specs.pdf https://www.icann.org/en/system/files/files/rdap-technical-implementation-guide-15feb19-en.pdf
+
+cat data/registrar-ids.xml | xq > data/registrar_ids.json

tools/launchpad_cve_puller/README.md

@@ -0,0 +1,136 @@
+# Launchpad CVE Puller:
+This program can be used to pull information for a CVE and all bugs/activity in Ubuntu's launchpad bug database.
+There is a confluence page that showcases a little bit more: https://kennasecurity.atlassian.net/wiki/spaces/EN/pages/2428895235/Ubuntu+-+Launchpad+hacking
+
+### Example Usage:
+Provide a CVE ID and the program will execute a query to launchpad for the CVE object. From this response it will parse the bugs_collection_link in the CVE object and will then send API queries for each bug id returned in the collection link. Each bug ID will then be used to query the activity for that bug. An example can be checked below.
+
+```
+bmcdev@BMCDEV-M-N4F3 launchpad_cve_puller % python3 cve_puller.py 'CVE-2015-8768'
+connect: (api.launchpad.net, 443)
+send: b'GET /1.0/ HTTP/1.1\r\nHost: api.launchpad.net\r\naccept: application/vnd.sun.wadl+xml\r\nuser-agent: lazr.restfulclient 0.14.4; oauth_consumer="just testing"\r\naccept-encoding: gzip, deflate\r\nif-none-match: "1762ac-5cfe85ab92a40-gzip"\r\nif-modified-since: Wed, 03 Nov 2021 20:35:45 GMT\r\nAuthorization: OAuth realm="OAuth", oauth_nonce="47903598616718557091636562636", oauth_timestamp="1636562636", oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="just%20testing", oauth_token="", oauth_signature="%26"\r\n\r\n'
+reply: 'HTTP/1.1 200 OK\r\n'
+header: Date: Wed, 10 Nov 2021 16:43:57 GMT
+header: Server: Apache
+header: Content-Location: index.wadl
+header: Vary: negotiate,accept,Accept-Encoding
+header: TCN: choice
+header: Last-Modified: Wed, 03 Nov 2021 20:35:45 GMT
+header: ETag: "1762ac-5cfe85ab92a40-gzip"
+header: Accept-Ranges: bytes
+header: Content-Encoding: gzip
+header: Transfer-Encoding: chunked
+header: Content-Type: application/vnd.sun.wadl+xml
+send: b'GET /1.0/ HTTP/1.1\r\nHost: api.launchpad.net\r\naccept: application/json\r\nuser-agent: lazr.restfulclient 0.14.4; oauth_consumer="just testing"\r\naccept-encoding: gzip, deflate\r\nif-none-match: "8bc-5cfe85ab92a40-gzip"\r\nif-modified-since: Wed, 03 Nov 2021 20:35:45 GMT\r\nAuthorization: OAuth realm="OAuth", oauth_nonce="99084209908414469401636562637", oauth_timestamp="1636562637", oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="just%20testing", oauth_token="", oauth_signature="%26"\r\n\r\n'
+reply: 'HTTP/1.1 200 OK\r\n'
+header: Date: Wed, 10 Nov 2021 16:43:58 GMT
+header: Server: Apache
+header: Content-Location: index.json
+header: Vary: negotiate,accept,Accept-Encoding
+header: TCN: choice
+header: Last-Modified: Wed, 03 Nov 2021 20:35:45 GMT
+header: ETag: "8bc-5cfe85ab92a40-gzip"
+header: Accept-Ranges: bytes
+header: Content-Encoding: gzip
+header: Content-Length: 420
+header: Content-Type: application/json
+Now looking up CVE: CVE-2015-8768
+------------------------------------------------------------------------------------
+CVE JSON: {
+    "self_link": "https://api.launchpad.net/devel/bugs/cve/2015-8768",
+    "web_link": "https://bugs.launchpad.net/bugs/cve/2015-8768",
+    "resource_type_link": "https://api.launchpad.net/devel/#cve",
+    "sequence": "2015-8768",
+    "status": "Candidate",
+    "description": "click/install.py in click does not require files in package filesystem tarballs to start with ./ (dot slash), which allows remote attackers to install an alternate security policy and gain privileges via a crafted package, as demonstrated by the test.mmrow app for Ubuntu phone.",
+    "date_created": "2016-01-12T20:09:02.711616+00:00",
+    "date_modified": "2017-10-03T09:02:44.564029+00:00",
+    "bugs_collection_link": "https://api.launchpad.net/devel/bugs/cve/2015-8768/bugs",
+    "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-8768",
+    "display_name": "CVE-2015-8768",
+    "title": "CVE-2015-8768 (Candidate)",
+    "http_etag": "\"3acf3f23b00798ecc308d3caa40d7187a1820651-10ec41981c2a6bb5d5f3acefe9424d7f33a43b19\""
+}
+------------------------------------------------------------------------------------
+Bugs JSON: {
+    "start": 0,
+    "total_size": 1,
+    "entries": [
+        {
+            "self_link": "https://api.launchpad.net/devel/bugs/1506467",
+            "web_link": "https://bugs.launchpad.net/bugs/1506467",
+            "resource_type_link": "https://api.launchpad.net/devel/#bug",
+            "id": 1506467,
+            "private": false,
+            "information_type": "Public Security",
+            "name": null,
+            "title": "click install does not ignore shipped files without leading './'",
+            "description": "The click install process does not filter out all illegitimate paths during the install process. For example, an app can ship '.click' in data.tar.gz which interferes with package installs. './.click/' is correctly filtered.",
+            "owner_link": "https://api.launchpad.net/devel/~jdstrand",
+            "bug_tasks_collection_link": "https://api.launchpad.net/devel/bugs/1506467/bug_tasks",
+            "duplicate_of_link": null,
+            "date_created": "2015-10-15T12:52:00.291948+00:00",
+            "activity_collection_link": "https://api.launchpad.net/devel/bugs/1506467/activity",
+            "subscriptions_collection_link": "https://api.launchpad.net/devel/bugs/1506467/subscriptions",
+            "date_last_updated": "2016-01-12T20:09:03.718917+00:00",
+            "who_made_private_link": null,
+            "date_made_private": null,
+            "heat": 260,
+            "bug_watches_collection_link": "https://api.launchpad.net/devel/bugs/1506467/bug_watches",
+            "cves_collection_link": "https://api.launchpad.net/devel/bugs/1506467/cves",
+            "duplicates_collection_link": "https://api.launchpad.net/devel/bugs/1506467/duplicates",
+            "attachments_collection_link": "https://api.launchpad.net/devel/bugs/1506467/attachments",
+            "security_related": true,
+            "latest_patch_uploaded": null,
+            "tags": [
+                "hotfix"
+            ],
+            "date_last_message": "2016-01-12T20:09:02.711616+00:00",
+            "number_of_duplicates": 0,
+            "message_count": 5,
+            "users_affected_count": 1,
+            "users_unaffected_count": 0,
+            "users_affected_collection_link": "https://api.launchpad.net/devel/bugs/1506467/users_affected",
+            "users_unaffected_collection_link": "https://api.launchpad.net/devel/bugs/1506467/users_unaffected",
+            "users_affected_count_with_dupes": 1,
+            "other_users_affected_count_with_dupes": 1,
+            "users_affected_with_dupes_collection_link": "https://api.launchpad.net/devel/bugs/1506467/users_affected_with_dupes",
+            "messages_collection_link": "https://api.launchpad.net/devel/bugs/1506467/messages",
+            "linked_branches_collection_link": "https://api.launchpad.net/devel/bugs/1506467/linked_branches",
+            "linked_merge_proposals_collection_link": "https://api.launchpad.net/devel/bugs/1506467/linked_merge_proposals",
+            "http_etag": "\"a9cdde051b66f2580427b6bde558947906183adf-b5dbe97cceed31acf9787d76f5b506bb21212702\""
+        }
+    ],
+    "resource_type_link": "https://api.launchpad.net/devel/#bug-page-resource"
+------------------------------------------------------------------------------------
+Activity for bug_id 1506467: {
+    "start": 0,
+    "total_size": 35,
+    "entries": [
+        {
+            "self_link": "https://api.launchpad.net/devel/bugs/1506467/activity",
+            "web_link": "https://bugs.launchpad.net/bugs/1506467/activity",
+            "resource_type_link": "https://api.launchpad.net/devel/#bug_activity",
+            "bug_link": "https://api.launchpad.net/devel/bugs/1506467",
+            "datechanged": "2015-10-15T12:52:00.291948+00:00",
+            "person_link": "https://api.launchpad.net/devel/~jdstrand",
+            "whatchanged": "bug",
+            "oldvalue": null,
+            "newvalue": null,
+            "message": "added bug",
+            "http_etag": "\"84b52aa2a611f243f9f72cb30bccc46924711668-c607ca46c77b9673130a54553f81fd2595304c96\""
+        },
+        {
+            "self_link": "https://api.launchpad.net/devel/bugs/1506467/activity",
+            "web_link": "https://bugs.launchpad.net/bugs/1506467/activity",
+            "resource_type_link": "https://api.launchpad.net/devel/#bug_activity",
+            "bug_link": "https://api.launchpad.net/devel/bugs/1506467",
+            "datechanged": "2015-10-15T12:52:20.335990+00:00",
+            "person_link": "https://api.launchpad.net/devel/~jdstrand",
+            "whatchanged": "bug",
+            "oldvalue": null,
+            "newvalue": null,
+            "message": "added subscriber Colin Watson",
+            "http_etag": "\"7561b6fcc937991c079216eb5dd7dcf8efe0d7c9-c607ca46c77b9673130a54553f81fd2595304c96\""
+        },
+```

tools/launchpad_cve_puller/cve_puller.py

@@ -0,0 +1,104 @@
+from launchpadlib.launchpad import Launchpad
+import requests
+import json
+import sys
+
+# for http logging support:
+import httplib2
+httplib2.debuglevel = 1
+
+
+class CvePuller:
+    def __init__(self):
+        self.cachedir = './launchpadlib/cachedir'
+        self.url = 'https://api.launchpad.net/devel'
+        self.launchpad = self.login()
+
+    def login(self):
+        return Launchpad.login_anonymously('just testing', 'production', self.cachedir)
+
+    def cves(self):
+        return self.launchpad.cves
+
+    def cve(self, sequence_id):
+        r = requests.get(
+            '{}/bugs/cve/{}'.format(self.url, sequence_id))
+        if r.status_code == 200:
+            return r.json()
+        else:
+            print("HTTP Code: {}".format(r.status_code))
+
+    def bug_from_cve(self, sequence_id):
+        try:
+            cve_json = self.cve(sequence_id)
+            bug_link = cve_json['bugs_collection_link']
+            return self.get_bug(bug_link)
+        except:
+            return 'Error occured while retrieving bug. Check HTTP status code for further information.'
+
+    def activity_from_bug(self, bug_id):
+        r = requests.get('{}/bugs/{}/activity'.format(self.url, bug_id))
+        if r.status_code == 200:
+            return r.json()
+        else:
+            print("HTTP Code: {}".format(r.status_code))
+
+    def get_bug(self, bug_collection_link):
+        r = requests.get(bug_collection_link)
+        if r.status_code == 200:
+            return r.json() 
+        else:
+            print("HTTP Code: {}".format(r.status_code))
+
+    def bugs_with_cves(self):
+        return self.launchpad.bugs.searchTasks(has_cve=True)
+
+    def iterate_cve_bugs(self, num):
+        bugs = []
+        for bug in self.bugs_with_cves()[0:num]:
+            bug_obj = bug.bug
+            bugs.append(bug_obj)
+
+            return bugs
+
+    def cve_id_to_seq_id(self, cve_id):
+        return cve_id[4:]
+
+
+def json_pp(data):
+    return json.dumps(data, indent=4)
+
+puller = CvePuller()
+
+cve_id = sys.argv[1]
+sequence_id = puller.cve_id_to_seq_id(cve_id)
+
+cve_json = puller.cve(sequence_id)
+cve_json_pp = json_pp(cve_json)
+
+bug_from_cve_json = puller.bug_from_cve(sequence_id)
+bug_from_cve_json_pp = json_pp(bug_from_cve_json)
+
+bug_ids = []
+
+for entry in bug_from_cve_json['entries']:
+    id = entry['id']
+    bug_ids.append(id)
+
+activity_results = []
+
+for bug_id in bug_ids:
+    activity = puller.activity_from_bug(bug_id)
+    activity_pp = json_pp(activity)
+
+print('Now looking up CVE: {}'.format(cve_id))
+print('-------' * 12)
+print('CVE JSON: {}'.format(cve_json_pp))
+print('-------' * 12)
+print('Bugs JSON: {}'.format(bug_from_cve_json_pp))
+print('-------' * 12)
+
+for bug_id in bug_ids:
+    activity = puller.activity_from_bug(bug_id)
+    activity_pp = json_pp(activity)
+    print('Activity for bug_id {}: {}'.format(bug_id, activity_pp))

tools/microsoft/app_build_number_cpe_mapper/exchange.rb

@@ -27,6 +27,7 @@ class MicrosoftExchangeReleaseInfo
     end
   end
 
+
   def parse_html(html)
     Nokogiri::HTML(html)
   end

tools/microsoft/bin/get_microsoft_exchange_release_builds.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require '../app_build_number_cpe_mapper/exchange.rb'
+require 'optparse'
+require 'json'
+
+def pretty_json(data_hash)
+  json = JSON.parse(data_hash.to_json)
+  JSON.pretty_generate(json)
+end
+
+def export_to_json(data, filename)
+  File.write(filename, data) 
+end
+
+def do_export(data_hash)
+  filename = './data/microsoft_exchange_release_info.json'
+  json = pretty_json(data_hash)
+  export_to_json(json, filename)
+  puts "----" * 12
+  puts "Succesfully Exported to #{filename}:"
+  puts "----" * 12
+  puts json
+  puts "----" * 12
+end
+
+@options = {}
+
+OptionParser.new do |opts|
+  opts.on("-e", "--export", "Export Microsoft Exchange Server release data to json file in ./data") do |export|
+    @options[:export] = export
+  end
+end.parse!
+
+scraper = MicrosoftExchangeReleaseInfo.new
+RestClient.log = 'stdout'
+
+if @options[:export]
+  data_hash = scraper.main
+  do_export(data_hash)
+end

tools/nmap_scanning/README.md

@@ -0,0 +1,19 @@
+# Nmap Go module: 
+### How to Build it:
+```
+# setup a go module pointing here:
+go mod init git.mcdevitt.tech/bpmcdevitt/security_research/tools/nmap_scanning
+# get the nmap library:
+go get github.com/Ullaakut/nmap/v2
+```
+
+### Create and test a new scan example
+```
+# create a directory
+mkdir basic_scan
+cd basic_scan
+# create a main.go file in that directory.
+go build .
+# run it
+go run .
+```

tools/nmap_scanning/basic_scan/main.go

@@ -0,0 +1,47 @@
+// taken from https://github.com/Ullaakut/nmap/blob/master/examples/basic_scan/main.go
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/Ullaakut/nmap/v2"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	// Equivalent to `/usr/local/bin/nmap -p 80,443,843 google.com facebook.com youtube.com`,
+	// with a 5 minute timeout.
+	scanner, err := nmap.NewScanner(
+		nmap.WithTargets("google.com", "facebook.com", "youtube.com"),
+		nmap.WithPorts("80,443,843"),
+		nmap.WithContext(ctx),
+	)
+	if err != nil {
+		log.Fatalf("unable to create nmap scanner: %v", err)
+	}
+
+	result, _, err := scanner.Run()
+	if err != nil {
+		log.Fatalf("unable to run nmap scan: %v", err)
+	}
+
+	// Use the results to print an example output
+	for _, host := range result.Hosts {
+		if len(host.Ports) == 0 || len(host.Addresses) == 0 {
+			continue
+		}
+
+		fmt.Printf("Host %q:\n", host.Addresses[0])
+
+		for _, port := range host.Ports {
+			fmt.Printf("\tPort %d/%s %s %s\n", port.ID, port.Protocol, port.State, port.Service.Name)
+		}
+	}
+
+	fmt.Printf("Nmap done: %d hosts up scanned in %.2f seconds\n", len(result.Hosts), result.Stats.Finished.Elapsed)
+}

tools/ubuntu_package_puller/distro_source_package_version_history.py

@@ -104,17 +104,20 @@ class UbuntuPackageHistory:
             published_sources = self.current_archive.getPublishedSources(source_name = self.source_package_name, distro_series=distro)
             return published_sources
 
+
     def source_package_version_history(self):
         sources = self.get_published_sources()
         results = []
         for source in sources:
             source_package_name = source.source_package_name
             source_package_version = source.source_package_version
-            results.append((source_package_name, source_package_version))
+            binary_file_urls = source.binaryFileUrls()
+            results.append((source_package_name, source_package_version, binary_file_urls))
 
         d = defaultdict(list)
-        for k, v in results:
-            d[k].append(v)
+        for k, v, c in results:
+            new_dict = {'source_package_version': v, 'binary_file_urls': c}
+            d[k].append(new_dict)
 
         return sorted(d.items())
 

tools/ubuntu_package_puller/ubuntu-package-puller.py

@@ -0,0 +1,20 @@
+import distro_source_package_version_history as p
+import argparse
+import json
+
+def main():
+	parser = argparse.ArgumentParser(description='Pull package versions from the Ubuntu archive.')
+	parser.add_argument('--source-package-name', required=True, help='Name of the source package')
+	parser.add_argument('--distro-version', required=True, help='Version of the distribution')
+	
+	args = parser.parse_args()
+	
+	package_name = args.source_package_name
+	distro_version = args.distro_version
+	results = p.UbuntuPackageHistory(source_package_name=package_name, distro_version=distro_version)
+	response = results.source_package_version_history()
+	print(json.dumps(response, indent=4))
+
+
+if __name__ == "__main__":
+	main()