Compare commits
No commits in common. "14b167e255426ad8ad1ce39df59b178b5d06364c" and "70985c7511649248e6dea756951942ca14f505cd" have entirely different histories.
14b167e255
...
70985c7511
12 changed files with 202 additions and 420 deletions
|
|
@ -1,13 +1,11 @@
|
|||
# syntax=docker/dockerfile:1
|
||||
FROM ruby:latest
|
||||
RUN apt-get update -qq && apt-get install -y nodejs postgresql-client less bzip2
|
||||
RUN apt-get update -qq && apt-get install -y nodejs postgresql-client less
|
||||
WORKDIR /data_importer
|
||||
COPY Gemfile /data_importer/Gemfile
|
||||
#COPY Gemfile.lock /data_importer/Gemfile.lock
|
||||
RUN bundle update
|
||||
COPY Gemfile.lock /data_importer/Gemfile.lock
|
||||
RUN bundle install
|
||||
ENV PAGER=less
|
||||
ENV LANG='UTF-8'
|
||||
|
||||
# Add a script to be executed every time the container starts.
|
||||
COPY entrypoint.sh /usr/bin/
|
||||
|
|
|
|||
9
Gemfile
9
Gemfile
|
|
@ -6,7 +6,6 @@ ruby ENV['RUBY_VERSION']
|
|||
|
||||
# Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
|
||||
gem 'actionpack'
|
||||
gem 'bzip2-ffi'
|
||||
gem 'faktory_worker_ruby'
|
||||
gem 'graphql'
|
||||
gem 'graphql-client'
|
||||
|
|
@ -18,15 +17,10 @@ gem 'retryable'
|
|||
gem 'rubocop'
|
||||
gem 'rubocop-graphql'
|
||||
gem 'rubocop-rails'
|
||||
gem 'rdoc'
|
||||
gem 'rexml', '~> 3.2.4'
|
||||
gem 'sass-rails'
|
||||
gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support
|
||||
gem 'twitter'
|
||||
gem 'mime-types-data', '~> 3.2024.0820'
|
||||
gem 'listen', '3.0.8'
|
||||
gem 'mutex_m'
|
||||
gem 'bigdecimal'
|
||||
|
||||
# Use postgres as the database for Active Record
|
||||
gem 'bulk_insert'
|
||||
gem 'git'
|
||||
|
|
@ -72,6 +66,7 @@ end
|
|||
|
||||
group :development do
|
||||
# Access an interactive console on exception pages or by calling 'console' anywhere in the code.
|
||||
gem 'listen', '>= 3.0.5', '< 3.2'
|
||||
gem 'web-console', '>= 3.3.0'
|
||||
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
|
||||
gem 'spring'
|
||||
|
|
|
|||
400
Gemfile.lock
400
Gemfile.lock
|
|
@ -1,6 +1,6 @@
|
|||
GIT
|
||||
remote: https://github.com/julianfssen/tweetkit.git
|
||||
revision: d93a9e18d5d7b487cc3e433ab640fa2d99a6069b
|
||||
revision: e9ff2e807089547548a3caeea24b06cbdb1defd3
|
||||
specs:
|
||||
tweetkit (0.2.0)
|
||||
faraday (~> 1.9.3)
|
||||
|
|
@ -10,98 +10,95 @@ GIT
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
actioncable (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
actioncable (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
nio4r (~> 2.0)
|
||||
websocket-driver (>= 0.6.1)
|
||||
actionmailbox (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
activejob (= 7.0.8.4)
|
||||
activerecord (= 7.0.8.4)
|
||||
activestorage (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
actionmailbox (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
activejob (= 7.0.2.3)
|
||||
activerecord (= 7.0.2.3)
|
||||
activestorage (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
mail (>= 2.7.1)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
actionmailer (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
actionview (= 7.0.8.4)
|
||||
activejob (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
actionmailer (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
actionview (= 7.0.2.3)
|
||||
activejob (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
mail (~> 2.5, >= 2.5.4)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
rails-dom-testing (~> 2.0)
|
||||
actionpack (7.0.8.4)
|
||||
actionview (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
rack (~> 2.0, >= 2.2.4)
|
||||
actionpack (7.0.2.3)
|
||||
actionview (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
rack (~> 2.0, >= 2.2.0)
|
||||
rack-test (>= 0.6.3)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
||||
actiontext (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
activerecord (= 7.0.8.4)
|
||||
activestorage (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
actiontext (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
activerecord (= 7.0.2.3)
|
||||
activestorage (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
globalid (>= 0.6.0)
|
||||
nokogiri (>= 1.8.5)
|
||||
actionview (7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
actionview (7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
builder (~> 3.1)
|
||||
erubi (~> 1.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
||||
activejob (7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
activejob (7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
globalid (>= 0.3.6)
|
||||
activemodel (7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
activerecord (7.0.8.4)
|
||||
activemodel (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
activestorage (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
activejob (= 7.0.8.4)
|
||||
activerecord (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
activemodel (7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
activerecord (7.0.2.3)
|
||||
activemodel (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
activestorage (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
activejob (= 7.0.2.3)
|
||||
activerecord (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
marcel (~> 1.0)
|
||||
mini_mime (>= 1.1.0)
|
||||
activesupport (7.0.8.4)
|
||||
activesupport (7.0.2.3)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
i18n (>= 1.6, < 2)
|
||||
minitest (>= 5.1)
|
||||
tzinfo (~> 2.0)
|
||||
addressable (2.8.7)
|
||||
public_suffix (>= 2.0.2, < 7.0)
|
||||
addressable (2.8.0)
|
||||
public_suffix (>= 2.0.2, < 5.0)
|
||||
archive-zip (0.12.0)
|
||||
io-like (~> 0.3.0)
|
||||
ast (2.4.2)
|
||||
awesome_print (1.9.2)
|
||||
base64 (0.2.0)
|
||||
bigdecimal (3.1.8)
|
||||
bindex (0.8.1)
|
||||
bootsnap (1.18.4)
|
||||
bootsnap (1.11.1)
|
||||
msgpack (~> 1.2)
|
||||
buftok (0.3.0)
|
||||
builder (3.3.0)
|
||||
buftok (0.2.0)
|
||||
builder (3.2.4)
|
||||
bulk_insert (1.9.0)
|
||||
activerecord (>= 3.2.0)
|
||||
byebug (11.1.3)
|
||||
bzip2-ffi (1.1.1)
|
||||
ffi (~> 1.0)
|
||||
capybara (3.40.0)
|
||||
capybara (3.36.0)
|
||||
addressable
|
||||
matrix
|
||||
mini_mime (>= 0.1.3)
|
||||
nokogiri (~> 1.11)
|
||||
nokogiri (~> 1.8)
|
||||
rack (>= 1.6.0)
|
||||
rack-test (>= 0.6.3)
|
||||
regexp_parser (>= 1.5, < 3.0)
|
||||
xpath (~> 3.2)
|
||||
childprocess (4.1.0)
|
||||
chromedriver-helper (2.1.1)
|
||||
archive-zip (~> 0.10)
|
||||
nokogiri (~> 1.8)
|
||||
|
|
@ -113,15 +110,16 @@ GEM
|
|||
coffee-script-source
|
||||
execjs
|
||||
coffee-script-source (1.12.2)
|
||||
concurrent-ruby (1.3.4)
|
||||
connection_pool (2.4.1)
|
||||
concurrent-ruby (1.1.10)
|
||||
connection_pool (2.2.5)
|
||||
crass (1.0.6)
|
||||
date (3.3.4)
|
||||
domain_name (0.6.20240107)
|
||||
digest (3.1.0)
|
||||
domain_name (0.5.20190701)
|
||||
unf (>= 0.0.5, < 1.0.0)
|
||||
equalizer (0.0.11)
|
||||
erubi (1.13.0)
|
||||
execjs (2.9.1)
|
||||
faktory_worker_ruby (2.0.0)
|
||||
erubi (1.10.0)
|
||||
execjs (2.8.1)
|
||||
faktory_worker_ruby (1.1.1)
|
||||
connection_pool (~> 2.2, >= 2.2.2)
|
||||
faraday (1.9.3)
|
||||
faraday-em_http (~> 1.0)
|
||||
|
|
@ -139,190 +137,169 @@ GEM
|
|||
faraday-em_synchrony (1.0.0)
|
||||
faraday-excon (1.1.0)
|
||||
faraday-httpclient (1.0.1)
|
||||
faraday-multipart (1.0.4)
|
||||
multipart-post (~> 2)
|
||||
faraday-net_http (1.0.2)
|
||||
faraday-multipart (1.0.3)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
faraday-net_http (1.0.1)
|
||||
faraday-net_http_persistent (1.2.0)
|
||||
faraday-patron (1.0.0)
|
||||
faraday-rack (1.0.0)
|
||||
faraday-retry (1.0.3)
|
||||
faraday_middleware (1.2.0)
|
||||
faraday (~> 1.0)
|
||||
ffi (1.17.0-x86_64-linux-gnu)
|
||||
ffi-compiler (1.3.2)
|
||||
ffi (>= 1.15.5)
|
||||
ffi (1.15.5)
|
||||
ffi-compiler (1.0.1)
|
||||
ffi (>= 1.0.0)
|
||||
rake
|
||||
fiber-storage (1.0.0)
|
||||
git (2.1.1)
|
||||
activesupport (>= 5.0)
|
||||
addressable (~> 2.8)
|
||||
process_executer (~> 1.1)
|
||||
git (1.11.0)
|
||||
rchardet (~> 1.8)
|
||||
globalid (1.2.1)
|
||||
activesupport (>= 6.1)
|
||||
graphql (2.3.14)
|
||||
base64
|
||||
fiber-storage
|
||||
graphql-client (0.23.0)
|
||||
globalid (1.0.0)
|
||||
activesupport (>= 5.0)
|
||||
graphql (1.13.12)
|
||||
graphql-client (0.17.0)
|
||||
activesupport (>= 3.0)
|
||||
graphql (>= 1.13.0)
|
||||
http (5.2.0)
|
||||
addressable (~> 2.8)
|
||||
base64 (~> 0.1)
|
||||
graphql (~> 1.10)
|
||||
http (4.4.1)
|
||||
addressable (~> 2.3)
|
||||
http-cookie (~> 1.0)
|
||||
http-form_data (~> 2.2)
|
||||
llhttp-ffi (~> 0.5.0)
|
||||
http-parser (~> 1.2.0)
|
||||
http-accept (1.7.0)
|
||||
http-cookie (1.0.7)
|
||||
http-cookie (1.0.4)
|
||||
domain_name (~> 0.5)
|
||||
http-form_data (2.3.0)
|
||||
i18n (1.14.5)
|
||||
http-parser (1.2.3)
|
||||
ffi-compiler (>= 1.0, < 2.0)
|
||||
http_parser.rb (0.6.0)
|
||||
i18n (1.10.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
interception (0.5)
|
||||
io-like (0.3.1)
|
||||
jbuilder (2.12.0)
|
||||
jbuilder (2.11.5)
|
||||
actionview (>= 5.0.0)
|
||||
activesupport (>= 5.0.0)
|
||||
json (2.7.2)
|
||||
language_server-protocol (3.17.0.3)
|
||||
listen (3.0.8)
|
||||
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||
rb-inotify (~> 0.9, >= 0.9.7)
|
||||
llhttp-ffi (0.5.0)
|
||||
ffi-compiler (~> 1.0)
|
||||
rake (~> 13.0)
|
||||
logger (1.6.0)
|
||||
loofah (2.22.0)
|
||||
loofah (2.16.0)
|
||||
crass (~> 1.0.2)
|
||||
nokogiri (>= 1.12.0)
|
||||
mail (2.8.1)
|
||||
nokogiri (>= 1.5.9)
|
||||
mail (2.7.1)
|
||||
mini_mime (>= 0.1.1)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
marcel (1.0.4)
|
||||
marcel (1.0.2)
|
||||
matrix (0.4.2)
|
||||
memoizable (0.4.2)
|
||||
thread_safe (~> 0.3, >= 0.3.1)
|
||||
method_source (1.1.0)
|
||||
mime-types (3.5.2)
|
||||
method_source (1.0.0)
|
||||
mime-types (3.4.1)
|
||||
mime-types-data (~> 3.2015)
|
||||
mime-types-data (3.2024.0820)
|
||||
mini_mime (1.1.5)
|
||||
minitest (5.25.1)
|
||||
msgpack (1.7.2)
|
||||
multipart-post (2.4.1)
|
||||
mutex_m (0.2.0)
|
||||
mime-types-data (3.2022.0105)
|
||||
mini_mime (1.1.2)
|
||||
minitest (5.15.0)
|
||||
msgpack (1.5.1)
|
||||
multipart-post (2.1.1)
|
||||
naught (1.1.0)
|
||||
net-imap (0.4.14)
|
||||
date
|
||||
net-imap (0.2.3)
|
||||
digest
|
||||
net-protocol
|
||||
net-pop (0.1.2)
|
||||
strscan
|
||||
net-pop (0.1.1)
|
||||
digest
|
||||
net-protocol
|
||||
net-protocol (0.2.2)
|
||||
timeout
|
||||
net-smtp (0.5.0)
|
||||
net-protocol (0.1.3)
|
||||
timeout
|
||||
net-smtp (0.3.1)
|
||||
digest
|
||||
net-protocol
|
||||
timeout
|
||||
netrc (0.11.0)
|
||||
nio4r (2.7.3)
|
||||
nokogiri (1.16.7-x86_64-linux)
|
||||
nio4r (2.5.8)
|
||||
nokogiri (1.13.4-x86_64-linux)
|
||||
racc (~> 1.4)
|
||||
parallel (1.26.3)
|
||||
parser (3.3.4.2)
|
||||
parallel (1.22.1)
|
||||
parser (3.1.2.0)
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
pg (1.5.7)
|
||||
process_executer (1.1.0)
|
||||
pry (0.14.2)
|
||||
pg (1.3.5)
|
||||
pry (0.13.1)
|
||||
coderay (~> 1.1)
|
||||
method_source (~> 1.0)
|
||||
pry-byebug (3.10.1)
|
||||
pry-byebug (3.9.0)
|
||||
byebug (~> 11.0)
|
||||
pry (>= 0.13, < 0.15)
|
||||
pry-doc (1.5.0)
|
||||
pry (~> 0.13.0)
|
||||
pry-doc (1.3.0)
|
||||
pry (~> 0.11)
|
||||
yard (~> 0.9.11)
|
||||
pry-rails (0.3.11)
|
||||
pry (>= 0.13.0)
|
||||
pry-rescue (1.6.0)
|
||||
pry-rails (0.3.9)
|
||||
pry (>= 0.10.4)
|
||||
pry-rescue (1.5.2)
|
||||
interception (>= 0.5)
|
||||
pry (>= 0.12.0)
|
||||
pry-theme (1.3.1)
|
||||
coderay (~> 1.1)
|
||||
psych (5.1.2)
|
||||
stringio
|
||||
public_suffix (6.0.1)
|
||||
public_suffix (4.0.7)
|
||||
puma (3.12.6)
|
||||
racc (1.8.1)
|
||||
rack (2.2.9)
|
||||
rack-test (2.1.0)
|
||||
rack (>= 1.3)
|
||||
rails (7.0.8.4)
|
||||
actioncable (= 7.0.8.4)
|
||||
actionmailbox (= 7.0.8.4)
|
||||
actionmailer (= 7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
actiontext (= 7.0.8.4)
|
||||
actionview (= 7.0.8.4)
|
||||
activejob (= 7.0.8.4)
|
||||
activemodel (= 7.0.8.4)
|
||||
activerecord (= 7.0.8.4)
|
||||
activestorage (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
racc (1.6.0)
|
||||
rack (2.2.3)
|
||||
rack-test (1.1.0)
|
||||
rack (>= 1.0, < 3)
|
||||
rails (7.0.2.3)
|
||||
actioncable (= 7.0.2.3)
|
||||
actionmailbox (= 7.0.2.3)
|
||||
actionmailer (= 7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
actiontext (= 7.0.2.3)
|
||||
actionview (= 7.0.2.3)
|
||||
activejob (= 7.0.2.3)
|
||||
activemodel (= 7.0.2.3)
|
||||
activerecord (= 7.0.2.3)
|
||||
activestorage (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
bundler (>= 1.15.0)
|
||||
railties (= 7.0.8.4)
|
||||
rails-dom-testing (2.2.0)
|
||||
activesupport (>= 5.0.0)
|
||||
minitest
|
||||
railties (= 7.0.2.3)
|
||||
rails-dom-testing (2.0.3)
|
||||
activesupport (>= 4.2.0)
|
||||
nokogiri (>= 1.6)
|
||||
rails-html-sanitizer (1.6.0)
|
||||
loofah (~> 2.21)
|
||||
nokogiri (~> 1.14)
|
||||
railties (7.0.8.4)
|
||||
actionpack (= 7.0.8.4)
|
||||
activesupport (= 7.0.8.4)
|
||||
rails-html-sanitizer (1.4.2)
|
||||
loofah (~> 2.3)
|
||||
railties (7.0.2.3)
|
||||
actionpack (= 7.0.2.3)
|
||||
activesupport (= 7.0.2.3)
|
||||
method_source
|
||||
rake (>= 12.2)
|
||||
thor (~> 1.0)
|
||||
zeitwerk (~> 2.5)
|
||||
rainbow (3.1.1)
|
||||
rake (13.2.1)
|
||||
rb-fsevent (0.11.2)
|
||||
rb-inotify (0.11.1)
|
||||
rake (13.0.6)
|
||||
rb-fsevent (0.11.1)
|
||||
rb-inotify (0.10.1)
|
||||
ffi (~> 1.0)
|
||||
rchardet (1.8.0)
|
||||
rdoc (6.7.0)
|
||||
psych (>= 4.0.0)
|
||||
regexp_parser (2.9.2)
|
||||
regexp_parser (2.3.0)
|
||||
rest-client (2.1.0)
|
||||
http-accept (>= 1.7.0, < 2.0)
|
||||
http-cookie (>= 1.0.2, < 2.0)
|
||||
mime-types (>= 1.16, < 4.0)
|
||||
netrc (~> 0.8)
|
||||
retryable (3.0.5)
|
||||
rexml (3.2.9)
|
||||
strscan
|
||||
rubocop (1.65.1)
|
||||
json (~> 2.3)
|
||||
language_server-protocol (>= 3.17.0)
|
||||
rexml (3.2.5)
|
||||
rubocop (1.28.1)
|
||||
parallel (~> 1.10)
|
||||
parser (>= 3.3.0.2)
|
||||
parser (>= 3.1.0.0)
|
||||
rainbow (>= 2.2.2, < 4.0)
|
||||
regexp_parser (>= 2.4, < 3.0)
|
||||
rexml (>= 3.2.5, < 4.0)
|
||||
rubocop-ast (>= 1.31.1, < 2.0)
|
||||
regexp_parser (>= 1.8, < 3.0)
|
||||
rexml
|
||||
rubocop-ast (>= 1.17.0, < 2.0)
|
||||
ruby-progressbar (~> 1.7)
|
||||
unicode-display_width (>= 2.4.0, < 3.0)
|
||||
rubocop-ast (1.32.1)
|
||||
parser (>= 3.3.1.0)
|
||||
rubocop-graphql (1.5.4)
|
||||
rubocop (>= 1.50, < 2)
|
||||
rubocop-rails (2.25.1)
|
||||
unicode-display_width (>= 1.4.0, < 3.0)
|
||||
rubocop-ast (1.17.0)
|
||||
parser (>= 3.1.1.0)
|
||||
rubocop-graphql (0.14.2)
|
||||
rubocop (>= 0.87, < 2)
|
||||
rubocop-rails (2.14.2)
|
||||
activesupport (>= 4.2.0)
|
||||
rack (>= 1.1)
|
||||
rubocop (>= 1.33.0, < 2.0)
|
||||
rubocop-ast (>= 1.31.1, < 2.0)
|
||||
ruby-progressbar (1.13.0)
|
||||
rubocop (>= 1.7.0, < 2.0)
|
||||
ruby-progressbar (1.11.0)
|
||||
ruby2_keywords (0.0.5)
|
||||
rubyzip (2.3.2)
|
||||
sass-rails (6.0.0)
|
||||
|
|
@ -335,62 +312,63 @@ GEM
|
|||
sprockets (> 3.0)
|
||||
sprockets-rails
|
||||
tilt
|
||||
selenium-webdriver (4.23.0)
|
||||
base64 (~> 0.2)
|
||||
logger (~> 1.4)
|
||||
selenium-webdriver (4.1.0)
|
||||
childprocess (>= 0.5, < 5.0)
|
||||
rexml (~> 3.2, >= 3.2.5)
|
||||
rubyzip (>= 1.2.2, < 3.0)
|
||||
websocket (~> 1.0)
|
||||
rubyzip (>= 1.2.2)
|
||||
simple_oauth (0.3.1)
|
||||
spring (2.1.1)
|
||||
spring-watcher-listen (2.0.1)
|
||||
listen (>= 2.7, < 4.0)
|
||||
spring (>= 1.2, < 3.0)
|
||||
sprockets (4.2.1)
|
||||
sprockets (4.0.3)
|
||||
concurrent-ruby (~> 1.0)
|
||||
rack (>= 2.2.4, < 4)
|
||||
sprockets-rails (3.5.2)
|
||||
actionpack (>= 6.1)
|
||||
activesupport (>= 6.1)
|
||||
rack (> 1, < 3)
|
||||
sprockets-rails (3.4.2)
|
||||
actionpack (>= 5.2)
|
||||
activesupport (>= 5.2)
|
||||
sprockets (>= 3.0.0)
|
||||
stringio (3.1.1)
|
||||
strscan (3.1.0)
|
||||
thor (1.3.1)
|
||||
strscan (3.0.1)
|
||||
thor (1.2.1)
|
||||
thread_safe (0.3.6)
|
||||
tilt (2.4.0)
|
||||
timeout (0.4.1)
|
||||
tilt (2.0.10)
|
||||
timeout (0.2.0)
|
||||
turbolinks (5.2.1)
|
||||
turbolinks-source (~> 5.2)
|
||||
turbolinks-source (5.2.0)
|
||||
twitter (8.1.0)
|
||||
addressable (~> 2.8)
|
||||
buftok (~> 0.3.0)
|
||||
twitter (7.0.0)
|
||||
addressable (~> 2.3)
|
||||
buftok (~> 0.2.0)
|
||||
equalizer (~> 0.0.11)
|
||||
http (~> 5.2)
|
||||
http-form_data (~> 2.3)
|
||||
llhttp-ffi (~> 0.5.0)
|
||||
http (~> 4.0)
|
||||
http-form_data (~> 2.0)
|
||||
http_parser.rb (~> 0.6.0)
|
||||
memoizable (~> 0.4.0)
|
||||
multipart-post (~> 2.4)
|
||||
naught (~> 1.1)
|
||||
multipart-post (~> 2.0)
|
||||
naught (~> 1.0)
|
||||
simple_oauth (~> 0.3.0)
|
||||
tzinfo (2.0.6)
|
||||
tzinfo (2.0.4)
|
||||
concurrent-ruby (~> 1.0)
|
||||
uglifier (4.2.0)
|
||||
execjs (>= 0.3.0, < 3)
|
||||
unicode-display_width (2.5.0)
|
||||
web-console (4.2.1)
|
||||
unf (0.1.4)
|
||||
unf_ext
|
||||
unf_ext (0.0.8.1)
|
||||
unicode-display_width (2.1.0)
|
||||
web-console (4.2.0)
|
||||
actionview (>= 6.0.0)
|
||||
activemodel (>= 6.0.0)
|
||||
bindex (>= 0.4.0)
|
||||
railties (>= 6.0.0)
|
||||
websocket (1.2.11)
|
||||
websocket-driver (0.7.6)
|
||||
webrick (1.7.0)
|
||||
websocket-driver (0.7.5)
|
||||
websocket-extensions (>= 0.1.0)
|
||||
websocket-extensions (0.1.5)
|
||||
xpath (3.2.0)
|
||||
nokogiri (~> 1.8)
|
||||
yard (0.9.36)
|
||||
zeitwerk (2.6.17)
|
||||
yard (0.9.27)
|
||||
webrick (~> 1.7.0)
|
||||
zeitwerk (2.5.4)
|
||||
|
||||
PLATFORMS
|
||||
x86_64-linux
|
||||
|
|
@ -398,11 +376,9 @@ PLATFORMS
|
|||
DEPENDENCIES
|
||||
actionpack
|
||||
awesome_print
|
||||
bigdecimal
|
||||
bootsnap (>= 1.1.0)
|
||||
bulk_insert
|
||||
byebug
|
||||
bzip2-ffi
|
||||
capybara (>= 2.15)
|
||||
chromedriver-helper
|
||||
coffee-rails (~> 4.2)
|
||||
|
|
@ -411,9 +387,7 @@ DEPENDENCIES
|
|||
graphql
|
||||
graphql-client
|
||||
jbuilder (~> 2.5)
|
||||
listen (= 3.0.8)
|
||||
mime-types-data (~> 3.2024.0820)
|
||||
mutex_m
|
||||
listen (>= 3.0.5, < 3.2)
|
||||
nokogiri
|
||||
pg
|
||||
pry
|
||||
|
|
@ -425,10 +399,8 @@ DEPENDENCIES
|
|||
puma (~> 3.11)
|
||||
rails (~> 7.0.0)
|
||||
railties
|
||||
rdoc
|
||||
rest-client
|
||||
retryable
|
||||
rexml (~> 3.2.4)
|
||||
rubocop
|
||||
rubocop-graphql
|
||||
rubocop-rails
|
||||
|
|
@ -444,7 +416,7 @@ DEPENDENCIES
|
|||
web-console (>= 3.3.0)
|
||||
|
||||
RUBY VERSION
|
||||
ruby 3.3.4p94
|
||||
ruby 3.1.2p20
|
||||
|
||||
BUNDLED WITH
|
||||
2.5.11
|
||||
2.3.7
|
||||
|
|
|
|||
|
|
@ -8,9 +8,6 @@ Rails.application.configure do
|
|||
# since you don't have to restart the web server when you make code changes.
|
||||
config.cache_classes = false
|
||||
|
||||
# make web console work with docker
|
||||
config.web_console.permissions = "0.0.0.0/0"
|
||||
|
||||
# Do not eager load code on boot.
|
||||
config.eager_load = false
|
||||
|
||||
|
|
@ -62,5 +59,5 @@ Rails.application.configure do
|
|||
|
||||
# Use an evented file watcher to asynchronously detect changes in source code,
|
||||
# routes, locales, etc. This feature depends on the listen gem.
|
||||
#config.file_watcher = ActiveSupport::EventedFileUpdateChecker
|
||||
config.file_watcher = ActiveSupport::EventedFileUpdateChecker
|
||||
end
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ end
|
|||
|
||||
def perform
|
||||
import_cves
|
||||
# import_gsds
|
||||
import_gsds
|
||||
import_github_pocs
|
||||
import_trickest_poc_cves
|
||||
import_inthewild_cve_exploits
|
||||
|
|
|
|||
|
|
@ -518,111 +518,29 @@ https://www.checkpoint.com/advisories/
|
|||
### Advisory
|
||||
https://www.google.com/about/appsecurity/research/
|
||||
#### Notes
|
||||
CNA json lists https://www.google.com/about/appsecurity/research/ which redirects to their bughunter portal. Navigating from there to this page shows some public reports: https://bughunters.google.com/report/reports
|
||||
CNA json lists https://www.google.com/about/appsecurity/research/ which redirects to their bughutner portal. Navigating from there to this page shows some public reports: https://bughunters.google.com/report/reports
|
||||
#### Captured Data Potential From Advisory
|
||||
|
||||
## CISA - ICS
|
||||
### Advisory
|
||||
https://www.cisa.gov/uscert/ics/alerts
|
||||
#### Notes
|
||||
They offer an rss feed to get the index of advisories. Each alert has different headings depending on the type, but there are some common ones:
|
||||
Summary heading has a table in it which seems to have info about the vuln. that table has these three headings: 'vulnerability_type, remotely_exploitable, impact'
|
||||
Mitigation heading as well seems to be common across alerts/advisories.
|
||||
They offer an rss feed to get the index of advisories
|
||||
#### Captured Data Potential From Advisory
|
||||
|
||||
## Cyber Security Works Pvt. Ltd.
|
||||
### Advisory
|
||||
https://cybersecurityworks.com/zerodays-vulnerability-list/
|
||||
#### Notes
|
||||
Paginated list of vulns. Each vuln has a Proof of Concept section that has screenshots and step by step ways to exploit
|
||||
Paginated list of vulns.
|
||||
#### Captured Data Potential From Advisory
|
||||
```
|
||||
{
|
||||
:bulletin_id => '2021-CSW-11-1054',
|
||||
:affected_vendor => 'X2CRM',
|
||||
:affected_product => 'X2CRM',
|
||||
:affected_version => 'Version 8.0',
|
||||
:status => 'fixed',
|
||||
:date => 'Dec 1, 2021',
|
||||
:bug_name => 'Stored Cross-Site Scripting',
|
||||
:cve_number => 'CVE-2021-33853',
|
||||
:cwe_id => 'CWE-79',
|
||||
:cvssv3_score => '6.1',
|
||||
:severity => 'Medium',
|
||||
:description => 'A Cross-Site Scripting (XSS) attack can cause arbitrary code (javascript) to run in a user’s browser while the browser is connected to a trusted website. As the vehicle for the attack, the application targets the users and not the application itself. Additionally, the XSS payload is executed when the user attempts to access any page of the CRM.',
|
||||
:proof_of_concept => {
|
||||
:issue => 'Stored Cross-Site Scripting',
|
||||
:step_1 => 'Login to the X2CRM as administrator.',
|
||||
:step_2 => 'Go to the “Administrator” tool and click on the “User Interface Management’’ submenu and select “Add Top Bar Link”.',
|
||||
:step_3 => 'Enter “<script>alert(“XSS”)</script>” in the “Link Name” field and submit the request.',
|
||||
:step_4 => 'By accessing any page within the CRM, the payload will be executed.'
|
||||
},
|
||||
:impact => [
|
||||
'Inject malicious code into the vulnerable variable and exploit the application through the Cross-Site Scripting vulnerability.',
|
||||
'Modify the code and get the session information of other users',
|
||||
'Compromise the user machine.'
|
||||
],
|
||||
:remediations => [
|
||||
'Perform context-sensitive encoding of entrusted input before echoing back to a browser using an encoding library throughout the application.',
|
||||
'Implement input validation for special characters on all the variables are reflected in the browser and stored in the database.',
|
||||
'Explicitly set the character set encoding for each page generated by the webserver.',
|
||||
'Encode dynamic output elements and filter specific characters in dynamic elements.'
|
||||
],
|
||||
:timeline => {
|
||||
:november_11_2021 => 'Discovered in X2CRM 8.0 Product',
|
||||
:december_1_2021 => 'CSW team reported to Vendor about the vulnerability.',
|
||||
:january_20_2022 => 'X2CRM team postponed the release of X2CRM 8.5.',
|
||||
:february_1_2022_01 => 'Vendor fixed the issue.',
|
||||
:february_1_2022_02 => 'CSW assigned the CVE Identifier (CVE-2021-33853).'
|
||||
},
|
||||
:discovered_by => 'Cyber Security Works Pvt. Ltd.'
|
||||
}
|
||||
```
|
||||
|
||||
## Dahua Technologies
|
||||
### Advisory
|
||||
https://www.dahuasecurity.com/support/cybersecurity/annoucementNotice
|
||||
#### Notes
|
||||
Paginated index of vulns.
|
||||
#### Captured Data Potential From Advisory
|
||||
```
|
||||
{
|
||||
:bulletin_id => 'DHCC-SA-201906-001',
|
||||
:first_published => 'June 10, 2019',
|
||||
:summary => 'Buffer overflow vulnerability found in some Dahua IP Camera devices. The vulnerability exits in the function of redirection display for serial port printing information, which can not be used by product basic functions. After an attacker logs in locally, this vulnerability can be exploited to cause device restart or arbitrary code execution. Dahua has identified the corresponding security problems in the static code auditing process, so it has gradually deleted this function, which is no longer available in the newer devices and softwares. Dahua has released versions of the affected products to fix the vulnerability.',
|
||||
:cve_id => 'CVE-2019-9676',
|
||||
:vulnerability_score => '7.8 (CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H)',
|
||||
:affected_products_and_fix_software => [
|
||||
{
|
||||
:affected_model => 'IPC-HFW1XXX',
|
||||
:firmware_version => 'Build before 2018/11',
|
||||
:fix_software => [
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
|
||||
]
|
||||
},
|
||||
{
|
||||
:affected_model => 'IPC-HDW1XXX',
|
||||
:firmware_version => 'Build before 2018/11',
|
||||
:fix_software => [
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
|
||||
]
|
||||
},
|
||||
{
|
||||
:affected_model => 'IPC-HFW2XXX',
|
||||
:firmware_version => 'Build before 2018/11',
|
||||
:fix_software => [
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
|
||||
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
|
||||
]
|
||||
}
|
||||
],
|
||||
:fix_software_download => 'Please download the corresponding fix software (or its newer version) as listed in the above table from Dahua website. Customers can also contact Dahua local technical support to obtain the fix software.'
|
||||
}
|
||||
```
|
||||
|
||||
## Debian Gnu/Linux
|
||||
### Advisory
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
version: "3.3"
|
||||
services:
|
||||
db:
|
||||
image: postgres
|
||||
|
|
@ -42,4 +43,4 @@ services:
|
|||
volumes:
|
||||
- ./crontab.yaml:/crontab.yaml
|
||||
depends_on:
|
||||
- faktory
|
||||
- faktory
|
||||
|
|
@ -24,14 +24,8 @@ class CveListImporter < GithubRepo
|
|||
|
||||
EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze
|
||||
|
||||
# Old Cve list url
|
||||
# def initialize
|
||||
# super(repo_url = 'https://github.com/CVEProject/cvelist.git', repo_path = '/data_importer/data/cve_list')
|
||||
# end
|
||||
|
||||
# New Cve list v5 url
|
||||
def initialize
|
||||
super(repo_url = 'https://github.com/CVEProject/cvelistV5.git', repo_path = '/data_importer/data/cve_list')
|
||||
super(repo_url = 'https://github.com/CVEProject/cvelist.git', repo_path = '/data_importer/data/cve_list')
|
||||
end
|
||||
|
||||
def list_jsons_for_year(year)
|
||||
|
|
|
|||
|
|
@ -15,23 +15,12 @@ class GithubRepo
|
|||
end
|
||||
|
||||
def pull_latest_changes
|
||||
`cd #{repo_path}; git stash; git pull;`
|
||||
`cd #{repo_path}; git pull;`
|
||||
puts "Now pulling latest changes from #{repo_path}"
|
||||
end
|
||||
|
||||
def read_json(filename)
|
||||
begin
|
||||
file = File.read(filename, encoding: 'utf-8')
|
||||
# Ensure the file content is valid UTF-8
|
||||
file.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
|
||||
JSON.parse(file, symbolize_names: true)
|
||||
rescue JSON::ParserError => e
|
||||
puts "Error parsing JSON: #{e}"
|
||||
rescue Encoding::InvalidByteSequenceError => e
|
||||
puts "Invalid byte sequence in file: #{e}"
|
||||
rescue JSON::GeneratorError => e
|
||||
puts "Error generating JSON: #{e}"
|
||||
end
|
||||
JSON.parse(File.read(filename), symbolize_names: true)
|
||||
end
|
||||
|
||||
def read_markdown(filename)
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
require '/data_importer/lib/oval_parser.rb'
|
||||
|
||||
class OracleLinuxOvalImporter
|
||||
attr_accessor :url, :filepath
|
||||
def initialize
|
||||
@url = 'https://linux.oracle.com/security/oval/com.oracle.elsa-all.xml.bz2'
|
||||
@filepath = '/data_importer/data/oracle_oval.xml.bz2'
|
||||
end
|
||||
|
||||
def get_file
|
||||
if File.exist? filepath
|
||||
puts "#{filepath} exists"
|
||||
else
|
||||
`wget -O #{filepath} #{url}`
|
||||
end
|
||||
end
|
||||
|
||||
def decompress_bz2
|
||||
Bzip2::FFI::Reader.read(filepath)
|
||||
end
|
||||
|
||||
def xml_doc
|
||||
get_file
|
||||
xml = decompress_bz2
|
||||
OvalParser.new(xml).doc
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -19,7 +19,7 @@ class JsonHelper
|
|||
end
|
||||
|
||||
def self.read_json_from_file(filename)
|
||||
JSON.parse(File.read(filename, :encoding => 'utf-8'), symbolize_names: true)
|
||||
JSON.parse(File.read(filename), symbolize_names: true)
|
||||
end
|
||||
|
||||
def self.read_json_from_url(url)
|
||||
|
|
|
|||
|
|
@ -1,54 +0,0 @@
|
|||
require 'nokogiri'
|
||||
|
||||
class OvalParser
|
||||
attr_accessor :data, :doc, :root
|
||||
def initialize(data)
|
||||
@data = data
|
||||
@doc = Nokogiri::XML(data)
|
||||
@root = doc.root
|
||||
end
|
||||
|
||||
def get_definitions
|
||||
root.xpath("//xmlns:definition")
|
||||
end
|
||||
|
||||
def get_def_ids
|
||||
definitions = get_definitions
|
||||
definitions.map { |d| d.xpath(".//@id").text }
|
||||
end
|
||||
|
||||
def oval_defs_for_id(id)
|
||||
definitions = get_definitions
|
||||
definitions.select do |definition|
|
||||
definition if definition.attributes.dig('id').value == id
|
||||
end
|
||||
end
|
||||
|
||||
def get_cve_def_info
|
||||
# largerly used this code here as a guide:
|
||||
# https://github.com/OpenSCAP/openscap-daemon/blob/1b9e9d4849573e1ce09728cc61c4564e5d605a8e/openscap_daemon/cve_scanner/generate_summary.py#L83-L104
|
||||
def_ids = get_def_ids
|
||||
def_ids.map do |id|
|
||||
oval_defs = oval_defs_for_id(id)
|
||||
oval_defs.map do |oval_def|
|
||||
title = oval_def.xpath(".//xmlns:title").text
|
||||
cve_meta = oval_def.xpath(".//xmlns:reference").select { |n| n.attributes.dig('source').value == 'CVE' }
|
||||
cve_ids = cve_meta.map { |cve_m| cve_m.attributes.dig('ref_id').value || 'None' }
|
||||
cve_urls = cve_meta.map { |cve_m| cve_m.attributes.dig('ref_url').value || 'None' }
|
||||
description = oval_def.xpath(".//xmlns:description").text
|
||||
severity = oval_def.xpath(".//xmlns:severity").text
|
||||
my_sample_data_hash = {
|
||||
:title => title,
|
||||
:cve_ids => cve_ids,
|
||||
:cve_urls => cve_urls,
|
||||
:description => description,
|
||||
:severity => severity
|
||||
}
|
||||
binding.pry
|
||||
my_sample_data_hash
|
||||
{}
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue