This commit is contained in:
booboy 2020-10-23 04:51:14 -05:00
commit 04e994800c
10566 changed files with 43082 additions and 0 deletions

View file

@ -0,0 +1,89 @@
# ./docker-compose.yml
version: '2.2'
services:
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
container_name: es01
environment:
- node.name=es01
- cluster.name=es-docker-cluster
- discovery.seed_hosts=es02,es03
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data01:/usr/share/elasticsearch/data
ports:
- 9200:9200
networks:
- elastic
es02:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
container_name: es02
environment:
- node.name=es02
- cluster.name=es-docker-cluster
- discovery.seed_hosts=es01,es03
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data02:/usr/share/elasticsearch/data
ports:
- 9201:9201
networks:
- elastic
es03:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
container_name: es03
environment:
- node.name=es03
- cluster.name=es-docker-cluster
- discovery.seed_hosts=es01,es02
- cluster.initial_master_nodes=es01,es02,es03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data03:/usr/share/elasticsearch/data
ports:
- 9202:9202
networks:
- elastic
kib01:
image: docker.elastic.co/kibana/kibana:7.9.3
container_name: kib01
ports:
- 5601:5601
environment:
ELASTICSEARCH_URL: http://es01:9200
ELASTICSEARCH_HOSTS: http://es01:9200
networks:
- elastic
volumes:
data01:
driver: local
data02:
driver: local
data03:
driver: local
networks:
elastic:
driver: bridge

View file

@ -0,0 +1,2 @@
# Domain registration tracker
This will be a container that will be devoted to keeping a running status of newly registered domain names from major registrars on the internet

View file

@ -0,0 +1,48 @@
## Step 1:
I used this python script https://github.com/x4nth055/pythoncode-tutorials/tree/master/web-scraping/html-table-extractor
to extract all of the tables from a redhat documentation URL.
```
# mk some datadirs
mkdir data
mkdir -p data/redhat8/security_api_results
mkdir -p data/redhat7/security_api_results
mkdir -p data/redhat6/security_api_results
# run the program to scrape and convert the data to csv
python html_table_extractor.py "https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html-single/package_manifest/index"
[+] Found a total of 9 tables.
[+] Saving table-1
[+] Saving table-2
[+] Saving table-3
[+] Saving table-4
[+] Saving table-5
[+] Saving table-6
[+] Saving table-7
[+] Saving table-8
[+] Saving table-9
```
This will create a csv file per table found in the html-single page result of a given distro.
## Step 2:
To process and de-duplicate all of the packages further, I created one master CSV file in each directory for each distro by doing the following filtering on the commandline against each table csv file.
```
cat table-* | cut -f 2 -d , | sort | uniq | sort > all_redhat7_rpm_package_manifest.csv
```
and this step was repeated for redhat 8, 7, and 6.
## Step 3:
After creating a list of each base set pkg name in the distro, we can then feed these pkgs into a query against the redhat security api using the following example loop:
```
cd data/redhat8
for pkg in $(cat all_redhat8_rpm_package_manifest.csv);
do curl "https://access.redhat.com/hydra/rest/securitydata/cve.json?package=$pkg" > ./security_api_results/${pkg}_security_api_results.json;
done
```
this will send out api calls to the security api asking for cves in json format of the given pkgname.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2013-4786","severity":"moderate","public_date":"2013-07-08T00:00:00Z","advisories":[],"bugzilla":"1326638","bugzilla_description":"CVE-2013-4786 OpenIPMI, freeipmi: Leakage of password hashes via RAKP authentication","cvss_score":5.0,"cvss_scoring_vector":"AV:N/AC:L/Au:N/C:P/I:N/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2013-4786.json"},{"CVE":"CVE-2011-4339","severity":"moderate","public_date":"2011-12-13T00:00:00Z","advisories":["RHSA-2011:1814","RHSA-2013:0123"],"bugzilla":"742837","bugzilla_description":"CVE-2011-4339 OpenIPMI: IPMI event daemon creates PID file with world writeable permissions","cvss_score":4.7,"cvss_scoring_vector":"AV:L/AC:M/Au:N/C:N/I:N/A:C","CWE":null,"affected_packages":["OpenIPMI-0:2.0.16-16.el5","ipmitool-0:1.8.11-12.el6_2.1"],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2011-4339.json"}]

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2020-16121","severity":"low","public_date":"2020-09-24T00:00:00Z","advisories":[],"bugzilla":"1884560","bugzilla_description":"CVE-2020-16121 PackageKit: local attacker could use this issue to learn the MIME type of any file on the system","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-200","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-16121.json","cvss3_scoring_vector":"CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:L/I:N/A:N","cvss3_score":"2.5"},{"CVE":"CVE-2020-16122","severity":"moderate","public_date":"2020-09-24T00:00:00Z","advisories":[],"bugzilla":"1884562","bugzilla_description":"CVE-2020-16122 PackageKit: local user could possibly use this issue to install untrusted packages","cvss_score":null,"cvss_scoring_vector":null,"CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-16122.json","cvss3_scoring_vector":"CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:N/I:H/A:N","cvss3_score":"4.7"},{"CVE":"CVE-2018-1106","severity":"moderate","public_date":"2018-04-23T14:00:00Z","advisories":["RHSA-2018:1224"],"bugzilla":"1565992","bugzilla_description":"CVE-2018-1106 PackageKit: authentication bypass allows to install signed packages without administrator privileges","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-287","affected_packages":["PackageKit-0:1.1.5-2.el7_5"],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2018-1106.json","cvss3_scoring_vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:N","cvss3_score":"5.5"},{"CVE":"CVE-2013-1764","severity":"low","public_date":"2013-07-30T00:00:00Z","advisories":[],"bugzilla":"1090771","bugzilla_description":"CVE-2013-1764 PackageKit: downgrade packages when using the Zypper backend","cvss_score":2.1,"cvss_scoring_vector":"AV:L/AC:L/Au:N/C:N/I:P/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2013-1764.json"},{"CVE":"CVE-2011-2515","severity":"low","public_date":"2011-07-01T00:00:00Z","advisories":[],"bugzilla":"717566","bugzilla_description":"CVE-2011-2515 PackageKit: installs unsigned RPM packages as though they were signed","cvss_score":1.5,"cvss_scoring_vector":"AV:L/AC:M/Au:S/C:N/I:P/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2011-2515.json"}]

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2020-16121","severity":"low","public_date":"2020-09-24T00:00:00Z","advisories":[],"bugzilla":"1884560","bugzilla_description":"CVE-2020-16121 PackageKit: local attacker could use this issue to learn the MIME type of any file on the system","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-200","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-16121.json","cvss3_scoring_vector":"CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:L/I:N/A:N","cvss3_score":"2.5"},{"CVE":"CVE-2020-16122","severity":"moderate","public_date":"2020-09-24T00:00:00Z","advisories":[],"bugzilla":"1884562","bugzilla_description":"CVE-2020-16122 PackageKit: local user could possibly use this issue to install untrusted packages","cvss_score":null,"cvss_scoring_vector":null,"CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-16122.json","cvss3_scoring_vector":"CVSS:3.1/AV:L/AC:H/PR:L/UI:N/S:U/C:N/I:H/A:N","cvss3_score":"4.7"},{"CVE":"CVE-2018-1106","severity":"moderate","public_date":"2018-04-23T14:00:00Z","advisories":["RHSA-2018:1224"],"bugzilla":"1565992","bugzilla_description":"CVE-2018-1106 PackageKit: authentication bypass allows to install signed packages without administrator privileges","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-287","affected_packages":["PackageKit-0:1.1.5-2.el7_5"],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2018-1106.json","cvss3_scoring_vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:N","cvss3_score":"5.5"},{"CVE":"CVE-2013-1764","severity":"low","public_date":"2013-07-30T00:00:00Z","advisories":[],"bugzilla":"1090771","bugzilla_description":"CVE-2013-1764 PackageKit: downgrade packages when using the Zypper backend","cvss_score":2.1,"cvss_scoring_vector":"AV:L/AC:L/Au:N/C:N/I:P/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2013-1764.json"},{"CVE":"CVE-2011-2515","severity":"low","public_date":"2011-07-01T00:00:00Z","advisories":[],"bugzilla":"717566","bugzilla_description":"CVE-2011-2515 PackageKit: installs unsigned RPM packages as though they were signed","cvss_score":1.5,"cvss_scoring_vector":"AV:L/AC:M/Au:S/C:N/I:P/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2011-2515.json"}]

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2009-2940","severity":"moderate","public_date":"2009-10-14T00:00:00Z","advisories":[],"bugzilla":"1516701","bugzilla_description":"CVE-2009-2940 PyGreSQL: Missing a function to call PQescapeStringConn()","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-79","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2009-2940.json","cvss3_scoring_vector":"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:C/C:L/I:L/A:N","cvss3_score":"5.4"}]

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2012-1502","severity":"low","public_date":"2012-03-08T00:00:00Z","advisories":[],"bugzilla":"801116","bugzilla_description":"CVE-2012-1502 PyPAM: Double free by processing passwords containing NULL-bytes","cvss_score":4.3,"cvss_scoring_vector":"AV:N/AC:M/Au:N/C:N/I:N/A:P","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2012-1502.json"}]

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2020-14343","severity":"moderate","public_date":"2020-07-22T00:00:00Z","advisories":[],"bugzilla":"1860466","bugzilla_description":"CVE-2020-14343 PyYAML: incomplete fix for CVE-2020-1747","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-20","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-14343.json","cvss3_scoring_vector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H","cvss3_score":"9.8"},{"CVE":"CVE-2020-1747","severity":"moderate","public_date":"2020-03-02T00:00:00Z","advisories":[],"bugzilla":"1807367","bugzilla_description":"CVE-2020-1747 PyYAML: arbitrary command execution through python/object/new when FullLoader is used","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-20","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2020-1747.json","cvss3_scoring_vector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H","cvss3_score":"9.8"},{"CVE":"CVE-2019-20477","severity":"moderate","public_date":"2019-11-18T00:00:00Z","advisories":[],"bugzilla":"1806005","bugzilla_description":"CVE-2019-20477 PyYAML: command execution through python/object/apply constructor in FullLoader","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-502","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2019-20477.json","cvss3_scoring_vector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H","cvss3_score":"9.8"},{"CVE":"CVE-2017-18342","severity":"moderate","public_date":"2018-06-27T00:00:00Z","advisories":[],"bugzilla":"1595743","bugzilla_description":"CVE-2017-18342 PyYAML: yaml.load() API could execute arbitrary code","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-20","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-18342.json","cvss3_scoring_vector":"CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H","cvss3_score":"8.1"}]

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2018-14036","severity":"moderate","public_date":"2018-07-13T00:00:00Z","advisories":[],"bugzilla":"1601019","bugzilla_description":"CVE-2018-14036 accountsservice: insufficient path check in user_change_icon_file_authorized_cb() in user.c","cvss_score":null,"cvss_scoring_vector":null,"CWE":"CWE-22","affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2018-14036.json","cvss3_scoring_vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N","cvss3_score":"5.0"},{"CVE":"CVE-2012-6655","severity":"low","public_date":"2012-09-17T00:00:00Z","advisories":[],"bugzilla":"1130538","bugzilla_description":"CVE-2012-6655 accountsservice: local encrypted password disclosure when changing password","cvss_score":1.9,"cvss_scoring_vector":"AV:L/AC:M/Au:N/C:P/I:N/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2012-6655.json"}]

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
[{"CVE":"CVE-2011-4578","severity":"low","public_date":"2011-07-30T00:00:00Z","advisories":[],"bugzilla":"760984","bugzilla_description":"CVE-2011-4578 acpid: Unsafe umask for actions executed by acpid","cvss_score":3.6,"cvss_scoring_vector":"AV:L/AC:L/Au:N/C:P/I:P/A:N","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2011-4578.json"},{"CVE":"CVE-2011-1159","severity":"low","public_date":"2011-01-19T00:00:00Z","advisories":[],"bugzilla":"688698","bugzilla_description":"CVE-2011-1159 acpid: blocked writes can lead to acpid daemon hang","cvss_score":2.1,"cvss_scoring_vector":"AV:L/AC:L/Au:N/C:N/I:N/A:P","CWE":null,"affected_packages":[],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2011-1159.json"},{"CVE":"CVE-2009-4033","severity":"important","public_date":"2009-12-07T00:00:00Z","advisories":["RHSA-2009:1642"],"bugzilla":"542926","bugzilla_description":"CVE-2009-4033 acpid: log file created with random permissions","cvss_score":6.9,"cvss_scoring_vector":"AV:L/AC:M/Au:N/C:C/I:C/A:C","CWE":null,"affected_packages":["acpid-0:1.0.4-9.el5_4.1"],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2009-4033.json"},{"CVE":"CVE-2009-0798","severity":"moderate","public_date":"2009-04-21T00:00:00Z","advisories":["RHSA-2009:0474"],"bugzilla":"494443","bugzilla_description":"CVE-2009-0798 acpid: too many open files DoS","cvss_score":2.1,"cvss_scoring_vector":"AV:L/AC:L/AU:N/C:N/I:N/A:P","CWE":null,"affected_packages":["acpid-0:1.0.1-3","acpid-0:1.0.2-4","acpid-0:1.0.4-7.el5_3.1","acpid-0:1.0.3-2.el4_7.1"],"resource_url":"https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2009-0798.json"}]

Some files were not shown because too many files have changed in this diff Show more