Compare commits

..

No commits in common. "14b167e255426ad8ad1ce39df59b178b5d06364c" and "70985c7511649248e6dea756951942ca14f505cd" have entirely different histories.

12 changed files with 202 additions and 420 deletions

View file

@ -1,13 +1,11 @@
# syntax=docker/dockerfile:1 # syntax=docker/dockerfile:1
FROM ruby:latest FROM ruby:latest
RUN apt-get update -qq && apt-get install -y nodejs postgresql-client less bzip2 RUN apt-get update -qq && apt-get install -y nodejs postgresql-client less
WORKDIR /data_importer WORKDIR /data_importer
COPY Gemfile /data_importer/Gemfile COPY Gemfile /data_importer/Gemfile
#COPY Gemfile.lock /data_importer/Gemfile.lock COPY Gemfile.lock /data_importer/Gemfile.lock
RUN bundle update
RUN bundle install RUN bundle install
ENV PAGER=less ENV PAGER=less
ENV LANG='UTF-8'
# Add a script to be executed every time the container starts. # Add a script to be executed every time the container starts.
COPY entrypoint.sh /usr/bin/ COPY entrypoint.sh /usr/bin/

View file

@ -6,7 +6,6 @@ ruby ENV['RUBY_VERSION']
# Bundle edge Rails instead: gem 'rails', github: 'rails/rails' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
gem 'actionpack' gem 'actionpack'
gem 'bzip2-ffi'
gem 'faktory_worker_ruby' gem 'faktory_worker_ruby'
gem 'graphql' gem 'graphql'
gem 'graphql-client' gem 'graphql-client'
@ -18,15 +17,10 @@ gem 'retryable'
gem 'rubocop' gem 'rubocop'
gem 'rubocop-graphql' gem 'rubocop-graphql'
gem 'rubocop-rails' gem 'rubocop-rails'
gem 'rdoc'
gem 'rexml', '~> 3.2.4'
gem 'sass-rails' gem 'sass-rails'
gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support gem 'tweetkit', github: 'julianfssen/tweetkit' # for twitter v2 api support
gem 'twitter' gem 'twitter'
gem 'mime-types-data', '~> 3.2024.0820'
gem 'listen', '3.0.8'
gem 'mutex_m'
gem 'bigdecimal'
# Use postgres as the database for Active Record # Use postgres as the database for Active Record
gem 'bulk_insert' gem 'bulk_insert'
gem 'git' gem 'git'
@ -72,6 +66,7 @@ end
group :development do group :development do
# Access an interactive console on exception pages or by calling 'console' anywhere in the code. # Access an interactive console on exception pages or by calling 'console' anywhere in the code.
gem 'listen', '>= 3.0.5', '< 3.2'
gem 'web-console', '>= 3.3.0' gem 'web-console', '>= 3.3.0'
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
gem 'spring' gem 'spring'

View file

@ -1,6 +1,6 @@
GIT GIT
remote: https://github.com/julianfssen/tweetkit.git remote: https://github.com/julianfssen/tweetkit.git
revision: d93a9e18d5d7b487cc3e433ab640fa2d99a6069b revision: e9ff2e807089547548a3caeea24b06cbdb1defd3
specs: specs:
tweetkit (0.2.0) tweetkit (0.2.0)
faraday (~> 1.9.3) faraday (~> 1.9.3)
@ -10,98 +10,95 @@ GIT
GEM GEM
remote: https://rubygems.org/ remote: https://rubygems.org/
specs: specs:
actioncable (7.0.8.4) actioncable (7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
nio4r (~> 2.0) nio4r (~> 2.0)
websocket-driver (>= 0.6.1) websocket-driver (>= 0.6.1)
actionmailbox (7.0.8.4) actionmailbox (7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
activejob (= 7.0.8.4) activejob (= 7.0.2.3)
activerecord (= 7.0.8.4) activerecord (= 7.0.2.3)
activestorage (= 7.0.8.4) activestorage (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
mail (>= 2.7.1) mail (>= 2.7.1)
net-imap net-imap
net-pop net-pop
net-smtp net-smtp
actionmailer (7.0.8.4) actionmailer (7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
actionview (= 7.0.8.4) actionview (= 7.0.2.3)
activejob (= 7.0.8.4) activejob (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
mail (~> 2.5, >= 2.5.4) mail (~> 2.5, >= 2.5.4)
net-imap net-imap
net-pop net-pop
net-smtp net-smtp
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
actionpack (7.0.8.4) actionpack (7.0.2.3)
actionview (= 7.0.8.4) actionview (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
rack (~> 2.0, >= 2.2.4) rack (~> 2.0, >= 2.2.0)
rack-test (>= 0.6.3) rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (7.0.8.4) actiontext (7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
activerecord (= 7.0.8.4) activerecord (= 7.0.2.3)
activestorage (= 7.0.8.4) activestorage (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
globalid (>= 0.6.0) globalid (>= 0.6.0)
nokogiri (>= 1.8.5) nokogiri (>= 1.8.5)
actionview (7.0.8.4) actionview (7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
builder (~> 3.1) builder (~> 3.1)
erubi (~> 1.4) erubi (~> 1.4)
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0)
activejob (7.0.8.4) activejob (7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
globalid (>= 0.3.6) globalid (>= 0.3.6)
activemodel (7.0.8.4) activemodel (7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
activerecord (7.0.8.4) activerecord (7.0.2.3)
activemodel (= 7.0.8.4) activemodel (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
activestorage (7.0.8.4) activestorage (7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
activejob (= 7.0.8.4) activejob (= 7.0.2.3)
activerecord (= 7.0.8.4) activerecord (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
marcel (~> 1.0) marcel (~> 1.0)
mini_mime (>= 1.1.0) mini_mime (>= 1.1.0)
activesupport (7.0.8.4) activesupport (7.0.2.3)
concurrent-ruby (~> 1.0, >= 1.0.2) concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2) i18n (>= 1.6, < 2)
minitest (>= 5.1) minitest (>= 5.1)
tzinfo (~> 2.0) tzinfo (~> 2.0)
addressable (2.8.7) addressable (2.8.0)
public_suffix (>= 2.0.2, < 7.0) public_suffix (>= 2.0.2, < 5.0)
archive-zip (0.12.0) archive-zip (0.12.0)
io-like (~> 0.3.0) io-like (~> 0.3.0)
ast (2.4.2) ast (2.4.2)
awesome_print (1.9.2) awesome_print (1.9.2)
base64 (0.2.0)
bigdecimal (3.1.8)
bindex (0.8.1) bindex (0.8.1)
bootsnap (1.18.4) bootsnap (1.11.1)
msgpack (~> 1.2) msgpack (~> 1.2)
buftok (0.3.0) buftok (0.2.0)
builder (3.3.0) builder (3.2.4)
bulk_insert (1.9.0) bulk_insert (1.9.0)
activerecord (>= 3.2.0) activerecord (>= 3.2.0)
byebug (11.1.3) byebug (11.1.3)
bzip2-ffi (1.1.1) capybara (3.36.0)
ffi (~> 1.0)
capybara (3.40.0)
addressable addressable
matrix matrix
mini_mime (>= 0.1.3) mini_mime (>= 0.1.3)
nokogiri (~> 1.11) nokogiri (~> 1.8)
rack (>= 1.6.0) rack (>= 1.6.0)
rack-test (>= 0.6.3) rack-test (>= 0.6.3)
regexp_parser (>= 1.5, < 3.0) regexp_parser (>= 1.5, < 3.0)
xpath (~> 3.2) xpath (~> 3.2)
childprocess (4.1.0)
chromedriver-helper (2.1.1) chromedriver-helper (2.1.1)
archive-zip (~> 0.10) archive-zip (~> 0.10)
nokogiri (~> 1.8) nokogiri (~> 1.8)
@ -113,15 +110,16 @@ GEM
coffee-script-source coffee-script-source
execjs execjs
coffee-script-source (1.12.2) coffee-script-source (1.12.2)
concurrent-ruby (1.3.4) concurrent-ruby (1.1.10)
connection_pool (2.4.1) connection_pool (2.2.5)
crass (1.0.6) crass (1.0.6)
date (3.3.4) digest (3.1.0)
domain_name (0.6.20240107) domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
equalizer (0.0.11) equalizer (0.0.11)
erubi (1.13.0) erubi (1.10.0)
execjs (2.9.1) execjs (2.8.1)
faktory_worker_ruby (2.0.0) faktory_worker_ruby (1.1.1)
connection_pool (~> 2.2, >= 2.2.2) connection_pool (~> 2.2, >= 2.2.2)
faraday (1.9.3) faraday (1.9.3)
faraday-em_http (~> 1.0) faraday-em_http (~> 1.0)
@ -139,190 +137,169 @@ GEM
faraday-em_synchrony (1.0.0) faraday-em_synchrony (1.0.0)
faraday-excon (1.1.0) faraday-excon (1.1.0)
faraday-httpclient (1.0.1) faraday-httpclient (1.0.1)
faraday-multipart (1.0.4) faraday-multipart (1.0.3)
multipart-post (~> 2) multipart-post (>= 1.2, < 3)
faraday-net_http (1.0.2) faraday-net_http (1.0.1)
faraday-net_http_persistent (1.2.0) faraday-net_http_persistent (1.2.0)
faraday-patron (1.0.0) faraday-patron (1.0.0)
faraday-rack (1.0.0) faraday-rack (1.0.0)
faraday-retry (1.0.3) faraday-retry (1.0.3)
faraday_middleware (1.2.0) faraday_middleware (1.2.0)
faraday (~> 1.0) faraday (~> 1.0)
ffi (1.17.0-x86_64-linux-gnu) ffi (1.15.5)
ffi-compiler (1.3.2) ffi-compiler (1.0.1)
ffi (>= 1.15.5) ffi (>= 1.0.0)
rake rake
fiber-storage (1.0.0) git (1.11.0)
git (2.1.1)
activesupport (>= 5.0)
addressable (~> 2.8)
process_executer (~> 1.1)
rchardet (~> 1.8) rchardet (~> 1.8)
globalid (1.2.1) globalid (1.0.0)
activesupport (>= 6.1) activesupport (>= 5.0)
graphql (2.3.14) graphql (1.13.12)
base64 graphql-client (0.17.0)
fiber-storage
graphql-client (0.23.0)
activesupport (>= 3.0) activesupport (>= 3.0)
graphql (>= 1.13.0) graphql (~> 1.10)
http (5.2.0) http (4.4.1)
addressable (~> 2.8) addressable (~> 2.3)
base64 (~> 0.1)
http-cookie (~> 1.0) http-cookie (~> 1.0)
http-form_data (~> 2.2) http-form_data (~> 2.2)
llhttp-ffi (~> 0.5.0) http-parser (~> 1.2.0)
http-accept (1.7.0) http-accept (1.7.0)
http-cookie (1.0.7) http-cookie (1.0.4)
domain_name (~> 0.5) domain_name (~> 0.5)
http-form_data (2.3.0) http-form_data (2.3.0)
i18n (1.14.5) http-parser (1.2.3)
ffi-compiler (>= 1.0, < 2.0)
http_parser.rb (0.6.0)
i18n (1.10.0)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
interception (0.5) interception (0.5)
io-like (0.3.1) io-like (0.3.1)
jbuilder (2.12.0) jbuilder (2.11.5)
actionview (>= 5.0.0) actionview (>= 5.0.0)
activesupport (>= 5.0.0) activesupport (>= 5.0.0)
json (2.7.2)
language_server-protocol (3.17.0.3)
listen (3.0.8) listen (3.0.8)
rb-fsevent (~> 0.9, >= 0.9.4) rb-fsevent (~> 0.9, >= 0.9.4)
rb-inotify (~> 0.9, >= 0.9.7) rb-inotify (~> 0.9, >= 0.9.7)
llhttp-ffi (0.5.0) loofah (2.16.0)
ffi-compiler (~> 1.0)
rake (~> 13.0)
logger (1.6.0)
loofah (2.22.0)
crass (~> 1.0.2) crass (~> 1.0.2)
nokogiri (>= 1.12.0) nokogiri (>= 1.5.9)
mail (2.8.1) mail (2.7.1)
mini_mime (>= 0.1.1) mini_mime (>= 0.1.1)
net-imap marcel (1.0.2)
net-pop
net-smtp
marcel (1.0.4)
matrix (0.4.2) matrix (0.4.2)
memoizable (0.4.2) memoizable (0.4.2)
thread_safe (~> 0.3, >= 0.3.1) thread_safe (~> 0.3, >= 0.3.1)
method_source (1.1.0) method_source (1.0.0)
mime-types (3.5.2) mime-types (3.4.1)
mime-types-data (~> 3.2015) mime-types-data (~> 3.2015)
mime-types-data (3.2024.0820) mime-types-data (3.2022.0105)
mini_mime (1.1.5) mini_mime (1.1.2)
minitest (5.25.1) minitest (5.15.0)
msgpack (1.7.2) msgpack (1.5.1)
multipart-post (2.4.1) multipart-post (2.1.1)
mutex_m (0.2.0)
naught (1.1.0) naught (1.1.0)
net-imap (0.4.14) net-imap (0.2.3)
date digest
net-protocol net-protocol
net-pop (0.1.2) strscan
net-pop (0.1.1)
digest
net-protocol net-protocol
net-protocol (0.2.2)
timeout timeout
net-smtp (0.5.0) net-protocol (0.1.3)
timeout
net-smtp (0.3.1)
digest
net-protocol net-protocol
timeout
netrc (0.11.0) netrc (0.11.0)
nio4r (2.7.3) nio4r (2.5.8)
nokogiri (1.16.7-x86_64-linux) nokogiri (1.13.4-x86_64-linux)
racc (~> 1.4) racc (~> 1.4)
parallel (1.26.3) parallel (1.22.1)
parser (3.3.4.2) parser (3.1.2.0)
ast (~> 2.4.1) ast (~> 2.4.1)
racc pg (1.3.5)
pg (1.5.7) pry (0.13.1)
process_executer (1.1.0)
pry (0.14.2)
coderay (~> 1.1) coderay (~> 1.1)
method_source (~> 1.0) method_source (~> 1.0)
pry-byebug (3.10.1) pry-byebug (3.9.0)
byebug (~> 11.0) byebug (~> 11.0)
pry (>= 0.13, < 0.15) pry (~> 0.13.0)
pry-doc (1.5.0) pry-doc (1.3.0)
pry (~> 0.11) pry (~> 0.11)
yard (~> 0.9.11) yard (~> 0.9.11)
pry-rails (0.3.11) pry-rails (0.3.9)
pry (>= 0.13.0) pry (>= 0.10.4)
pry-rescue (1.6.0) pry-rescue (1.5.2)
interception (>= 0.5) interception (>= 0.5)
pry (>= 0.12.0) pry (>= 0.12.0)
pry-theme (1.3.1) pry-theme (1.3.1)
coderay (~> 1.1) coderay (~> 1.1)
psych (5.1.2) public_suffix (4.0.7)
stringio
public_suffix (6.0.1)
puma (3.12.6) puma (3.12.6)
racc (1.8.1) racc (1.6.0)
rack (2.2.9) rack (2.2.3)
rack-test (2.1.0) rack-test (1.1.0)
rack (>= 1.3) rack (>= 1.0, < 3)
rails (7.0.8.4) rails (7.0.2.3)
actioncable (= 7.0.8.4) actioncable (= 7.0.2.3)
actionmailbox (= 7.0.8.4) actionmailbox (= 7.0.2.3)
actionmailer (= 7.0.8.4) actionmailer (= 7.0.2.3)
actionpack (= 7.0.8.4) actionpack (= 7.0.2.3)
actiontext (= 7.0.8.4) actiontext (= 7.0.2.3)
actionview (= 7.0.8.4) actionview (= 7.0.2.3)
activejob (= 7.0.8.4) activejob (= 7.0.2.3)
activemodel (= 7.0.8.4) activemodel (= 7.0.2.3)
activerecord (= 7.0.8.4) activerecord (= 7.0.2.3)
activestorage (= 7.0.8.4) activestorage (= 7.0.2.3)
activesupport (= 7.0.8.4) activesupport (= 7.0.2.3)
bundler (>= 1.15.0) bundler (>= 1.15.0)
railties (= 7.0.8.4) railties (= 7.0.2.3)
rails-dom-testing (2.2.0) rails-dom-testing (2.0.3)
activesupport (>= 5.0.0) activesupport (>= 4.2.0)
minitest
nokogiri (>= 1.6) nokogiri (>= 1.6)
rails-html-sanitizer (1.6.0) rails-html-sanitizer (1.4.2)
loofah (~> 2.21) loofah (~> 2.3)
nokogiri (~> 1.14) railties (7.0.2.3)
railties (7.0.8.4) actionpack (= 7.0.2.3)
actionpack (= 7.0.8.4) activesupport (= 7.0.2.3)
activesupport (= 7.0.8.4)
method_source method_source
rake (>= 12.2) rake (>= 12.2)
thor (~> 1.0) thor (~> 1.0)
zeitwerk (~> 2.5) zeitwerk (~> 2.5)
rainbow (3.1.1) rainbow (3.1.1)
rake (13.2.1) rake (13.0.6)
rb-fsevent (0.11.2) rb-fsevent (0.11.1)
rb-inotify (0.11.1) rb-inotify (0.10.1)
ffi (~> 1.0) ffi (~> 1.0)
rchardet (1.8.0) rchardet (1.8.0)
rdoc (6.7.0) regexp_parser (2.3.0)
psych (>= 4.0.0)
regexp_parser (2.9.2)
rest-client (2.1.0) rest-client (2.1.0)
http-accept (>= 1.7.0, < 2.0) http-accept (>= 1.7.0, < 2.0)
http-cookie (>= 1.0.2, < 2.0) http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0) mime-types (>= 1.16, < 4.0)
netrc (~> 0.8) netrc (~> 0.8)
retryable (3.0.5) retryable (3.0.5)
rexml (3.2.9) rexml (3.2.5)
strscan rubocop (1.28.1)
rubocop (1.65.1)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10) parallel (~> 1.10)
parser (>= 3.3.0.2) parser (>= 3.1.0.0)
rainbow (>= 2.2.2, < 4.0) rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 2.4, < 3.0) regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0) rexml
rubocop-ast (>= 1.31.1, < 2.0) rubocop-ast (>= 1.17.0, < 2.0)
ruby-progressbar (~> 1.7) ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0) unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.32.1) rubocop-ast (1.17.0)
parser (>= 3.3.1.0) parser (>= 3.1.1.0)
rubocop-graphql (1.5.4) rubocop-graphql (0.14.2)
rubocop (>= 1.50, < 2) rubocop (>= 0.87, < 2)
rubocop-rails (2.25.1) rubocop-rails (2.14.2)
activesupport (>= 4.2.0) activesupport (>= 4.2.0)
rack (>= 1.1) rack (>= 1.1)
rubocop (>= 1.33.0, < 2.0) rubocop (>= 1.7.0, < 2.0)
rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (1.11.0)
ruby-progressbar (1.13.0)
ruby2_keywords (0.0.5) ruby2_keywords (0.0.5)
rubyzip (2.3.2) rubyzip (2.3.2)
sass-rails (6.0.0) sass-rails (6.0.0)
@ -335,62 +312,63 @@ GEM
sprockets (> 3.0) sprockets (> 3.0)
sprockets-rails sprockets-rails
tilt tilt
selenium-webdriver (4.23.0) selenium-webdriver (4.1.0)
base64 (~> 0.2) childprocess (>= 0.5, < 5.0)
logger (~> 1.4)
rexml (~> 3.2, >= 3.2.5) rexml (~> 3.2, >= 3.2.5)
rubyzip (>= 1.2.2, < 3.0) rubyzip (>= 1.2.2)
websocket (~> 1.0)
simple_oauth (0.3.1) simple_oauth (0.3.1)
spring (2.1.1) spring (2.1.1)
spring-watcher-listen (2.0.1) spring-watcher-listen (2.0.1)
listen (>= 2.7, < 4.0) listen (>= 2.7, < 4.0)
spring (>= 1.2, < 3.0) spring (>= 1.2, < 3.0)
sprockets (4.2.1) sprockets (4.0.3)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
rack (>= 2.2.4, < 4) rack (> 1, < 3)
sprockets-rails (3.5.2) sprockets-rails (3.4.2)
actionpack (>= 6.1) actionpack (>= 5.2)
activesupport (>= 6.1) activesupport (>= 5.2)
sprockets (>= 3.0.0) sprockets (>= 3.0.0)
stringio (3.1.1) strscan (3.0.1)
strscan (3.1.0) thor (1.2.1)
thor (1.3.1)
thread_safe (0.3.6) thread_safe (0.3.6)
tilt (2.4.0) tilt (2.0.10)
timeout (0.4.1) timeout (0.2.0)
turbolinks (5.2.1) turbolinks (5.2.1)
turbolinks-source (~> 5.2) turbolinks-source (~> 5.2)
turbolinks-source (5.2.0) turbolinks-source (5.2.0)
twitter (8.1.0) twitter (7.0.0)
addressable (~> 2.8) addressable (~> 2.3)
buftok (~> 0.3.0) buftok (~> 0.2.0)
equalizer (~> 0.0.11) equalizer (~> 0.0.11)
http (~> 5.2) http (~> 4.0)
http-form_data (~> 2.3) http-form_data (~> 2.0)
llhttp-ffi (~> 0.5.0) http_parser.rb (~> 0.6.0)
memoizable (~> 0.4.0) memoizable (~> 0.4.0)
multipart-post (~> 2.4) multipart-post (~> 2.0)
naught (~> 1.1) naught (~> 1.0)
simple_oauth (~> 0.3.0) simple_oauth (~> 0.3.0)
tzinfo (2.0.6) tzinfo (2.0.4)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
uglifier (4.2.0) uglifier (4.2.0)
execjs (>= 0.3.0, < 3) execjs (>= 0.3.0, < 3)
unicode-display_width (2.5.0) unf (0.1.4)
web-console (4.2.1) unf_ext
unf_ext (0.0.8.1)
unicode-display_width (2.1.0)
web-console (4.2.0)
actionview (>= 6.0.0) actionview (>= 6.0.0)
activemodel (>= 6.0.0) activemodel (>= 6.0.0)
bindex (>= 0.4.0) bindex (>= 0.4.0)
railties (>= 6.0.0) railties (>= 6.0.0)
websocket (1.2.11) webrick (1.7.0)
websocket-driver (0.7.6) websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0) websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5) websocket-extensions (0.1.5)
xpath (3.2.0) xpath (3.2.0)
nokogiri (~> 1.8) nokogiri (~> 1.8)
yard (0.9.36) yard (0.9.27)
zeitwerk (2.6.17) webrick (~> 1.7.0)
zeitwerk (2.5.4)
PLATFORMS PLATFORMS
x86_64-linux x86_64-linux
@ -398,11 +376,9 @@ PLATFORMS
DEPENDENCIES DEPENDENCIES
actionpack actionpack
awesome_print awesome_print
bigdecimal
bootsnap (>= 1.1.0) bootsnap (>= 1.1.0)
bulk_insert bulk_insert
byebug byebug
bzip2-ffi
capybara (>= 2.15) capybara (>= 2.15)
chromedriver-helper chromedriver-helper
coffee-rails (~> 4.2) coffee-rails (~> 4.2)
@ -411,9 +387,7 @@ DEPENDENCIES
graphql graphql
graphql-client graphql-client
jbuilder (~> 2.5) jbuilder (~> 2.5)
listen (= 3.0.8) listen (>= 3.0.5, < 3.2)
mime-types-data (~> 3.2024.0820)
mutex_m
nokogiri nokogiri
pg pg
pry pry
@ -425,10 +399,8 @@ DEPENDENCIES
puma (~> 3.11) puma (~> 3.11)
rails (~> 7.0.0) rails (~> 7.0.0)
railties railties
rdoc
rest-client rest-client
retryable retryable
rexml (~> 3.2.4)
rubocop rubocop
rubocop-graphql rubocop-graphql
rubocop-rails rubocop-rails
@ -444,7 +416,7 @@ DEPENDENCIES
web-console (>= 3.3.0) web-console (>= 3.3.0)
RUBY VERSION RUBY VERSION
ruby 3.3.4p94 ruby 3.1.2p20
BUNDLED WITH BUNDLED WITH
2.5.11 2.3.7

View file

@ -8,9 +8,6 @@ Rails.application.configure do
# since you don't have to restart the web server when you make code changes. # since you don't have to restart the web server when you make code changes.
config.cache_classes = false config.cache_classes = false
# make web console work with docker
config.web_console.permissions = "0.0.0.0/0"
# Do not eager load code on boot. # Do not eager load code on boot.
config.eager_load = false config.eager_load = false
@ -62,5 +59,5 @@ Rails.application.configure do
# Use an evented file watcher to asynchronously detect changes in source code, # Use an evented file watcher to asynchronously detect changes in source code,
# routes, locales, etc. This feature depends on the listen gem. # routes, locales, etc. This feature depends on the listen gem.
#config.file_watcher = ActiveSupport::EventedFileUpdateChecker config.file_watcher = ActiveSupport::EventedFileUpdateChecker
end end

View file

@ -26,7 +26,7 @@ end
def perform def perform
import_cves import_cves
# import_gsds import_gsds
import_github_pocs import_github_pocs
import_trickest_poc_cves import_trickest_poc_cves
import_inthewild_cve_exploits import_inthewild_cve_exploits

View file

@ -518,111 +518,29 @@ https://www.checkpoint.com/advisories/
### Advisory ### Advisory
https://www.google.com/about/appsecurity/research/ https://www.google.com/about/appsecurity/research/
#### Notes #### Notes
CNA json lists https://www.google.com/about/appsecurity/research/ which redirects to their bughunter portal. Navigating from there to this page shows some public reports: https://bughunters.google.com/report/reports CNA json lists https://www.google.com/about/appsecurity/research/ which redirects to their bughutner portal. Navigating from there to this page shows some public reports: https://bughunters.google.com/report/reports
#### Captured Data Potential From Advisory #### Captured Data Potential From Advisory
## CISA - ICS ## CISA - ICS
### Advisory ### Advisory
https://www.cisa.gov/uscert/ics/alerts https://www.cisa.gov/uscert/ics/alerts
#### Notes #### Notes
They offer an rss feed to get the index of advisories. Each alert has different headings depending on the type, but there are some common ones: They offer an rss feed to get the index of advisories
Summary heading has a table in it which seems to have info about the vuln. that table has these three headings: 'vulnerability_type, remotely_exploitable, impact'
Mitigation heading as well seems to be common across alerts/advisories.
#### Captured Data Potential From Advisory #### Captured Data Potential From Advisory
## Cyber Security Works Pvt. Ltd. ## Cyber Security Works Pvt. Ltd.
### Advisory ### Advisory
https://cybersecurityworks.com/zerodays-vulnerability-list/ https://cybersecurityworks.com/zerodays-vulnerability-list/
#### Notes #### Notes
Paginated list of vulns. Each vuln has a Proof of Concept section that has screenshots and step by step ways to exploit Paginated list of vulns.
#### Captured Data Potential From Advisory #### Captured Data Potential From Advisory
```
{
:bulletin_id => '2021-CSW-11-1054',
:affected_vendor => 'X2CRM',
:affected_product => 'X2CRM',
:affected_version => 'Version 8.0',
:status => 'fixed',
:date => 'Dec 1, 2021',
:bug_name => 'Stored Cross-Site Scripting',
:cve_number => 'CVE-2021-33853',
:cwe_id => 'CWE-79',
:cvssv3_score => '6.1',
:severity => 'Medium',
:description => 'A Cross-Site Scripting (XSS) attack can cause arbitrary code (javascript) to run in a users browser while the browser is connected to a trusted website. As the vehicle for the attack, the application targets the users and not the application itself. Additionally, the XSS payload is executed when the user attempts to access any page of the CRM.',
:proof_of_concept => {
:issue => 'Stored Cross-Site Scripting',
:step_1 => 'Login to the X2CRM as administrator.',
:step_2 => 'Go to the “Administrator” tool and click on the “User Interface Management submenu and select “Add Top Bar Link”.',
:step_3 => 'Enter “<script>alert(XSS)</script>” in the “Link Name” field and submit the request.',
:step_4 => 'By accessing any page within the CRM, the payload will be executed.'
},
:impact => [
'Inject malicious code into the vulnerable variable and exploit the application through the Cross-Site Scripting vulnerability.',
'Modify the code and get the session information of other users',
'Compromise the user machine.'
],
:remediations => [
'Perform context-sensitive encoding of entrusted input before echoing back to a browser using an encoding library throughout the application.',
'Implement input validation for special characters on all the variables are reflected in the browser and stored in the database.',
'Explicitly set the character set encoding for each page generated by the webserver.',
'Encode dynamic output elements and filter specific characters in dynamic elements.'
],
:timeline => {
:november_11_2021 => 'Discovered in X2CRM 8.0 Product',
:december_1_2021 => 'CSW team reported to Vendor about the vulnerability.',
:january_20_2022 => 'X2CRM team postponed the release of X2CRM 8.5.',
:february_1_2022_01 => 'Vendor fixed the issue.',
:february_1_2022_02 => 'CSW assigned the CVE Identifier (CVE-2021-33853).'
},
:discovered_by => 'Cyber Security Works Pvt. Ltd.'
}
```
## Dahua Technologies ## Dahua Technologies
### Advisory ### Advisory
https://www.dahuasecurity.com/support/cybersecurity/annoucementNotice https://www.dahuasecurity.com/support/cybersecurity/annoucementNotice
#### Notes #### Notes
Paginated index of vulns. Paginated index of vulns.
#### Captured Data Potential From Advisory #### Captured Data Potential From Advisory
```
{
:bulletin_id => 'DHCC-SA-201906-001',
:first_published => 'June 10, 2019',
:summary => 'Buffer overflow vulnerability found in some Dahua IP Camera devices. The vulnerability exits in the function of redirection display for serial port printing information, which can not be used by product basic functions. After an attacker logs in locally, this vulnerability can be exploited to cause device restart or arbitrary code execution. Dahua has identified the corresponding security problems in the static code auditing process, so it has gradually deleted this function, which is no longer available in the newer devices and softwares. Dahua has released versions of the affected products to fix the vulnerability.',
:cve_id => 'CVE-2019-9676',
:vulnerability_score => '7.8 (CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H)',
:affected_products_and_fix_software => [
{
:affected_model => 'IPC-HFW1XXX',
:firmware_version => 'Build before 2018/11',
:fix_software => [
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
]
},
{
:affected_model => 'IPC-HDW1XXX',
:firmware_version => 'Build before 2018/11',
:fix_software => [
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
]
},
{
:affected_model => 'IPC-HFW2XXX',
:firmware_version => 'Build before 2018/11',
:fix_software => [
'DH_IPC-HX1XXX-Eris_EngSpnPrt_P_V2.622.0000000.8.R.190420',
'DH_IPC-HX1XXX-Eris_EngSpnPrt_N_V2.622.0000000.8.R.190420',
'DH_IPC-HX2XXX-Eris_Chn_PN_V2.622.0000000.8.R.190420'
]
}
],
:fix_software_download => 'Please download the corresponding fix software (or its newer version) as listed in the above table from Dahua website. Customers can also contact Dahua local technical support to obtain the fix software.'
}
```
## Debian Gnu/Linux ## Debian Gnu/Linux
### Advisory ### Advisory

View file

@ -1,3 +1,4 @@
version: "3.3"
services: services:
db: db:
image: postgres image: postgres

View file

@ -24,14 +24,8 @@ class CveListImporter < GithubRepo
EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze EMPTY_HASH = EXPECTED_KEYS.map { |k| [k, nil] }.to_h.freeze
# Old Cve list url
# def initialize
# super(repo_url = 'https://github.com/CVEProject/cvelist.git', repo_path = '/data_importer/data/cve_list')
# end
# New Cve list v5 url
def initialize def initialize
super(repo_url = 'https://github.com/CVEProject/cvelistV5.git', repo_path = '/data_importer/data/cve_list') super(repo_url = 'https://github.com/CVEProject/cvelist.git', repo_path = '/data_importer/data/cve_list')
end end
def list_jsons_for_year(year) def list_jsons_for_year(year)

View file

@ -15,23 +15,12 @@ class GithubRepo
end end
def pull_latest_changes def pull_latest_changes
`cd #{repo_path}; git stash; git pull;` `cd #{repo_path}; git pull;`
puts "Now pulling latest changes from #{repo_path}" puts "Now pulling latest changes from #{repo_path}"
end end
def read_json(filename) def read_json(filename)
begin JSON.parse(File.read(filename), symbolize_names: true)
file = File.read(filename, encoding: 'utf-8')
# Ensure the file content is valid UTF-8
file.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
JSON.parse(file, symbolize_names: true)
rescue JSON::ParserError => e
puts "Error parsing JSON: #{e}"
rescue Encoding::InvalidByteSequenceError => e
puts "Invalid byte sequence in file: #{e}"
rescue JSON::GeneratorError => e
puts "Error generating JSON: #{e}"
end
end end
def read_markdown(filename) def read_markdown(filename)

View file

@ -1,28 +0,0 @@
require '/data_importer/lib/oval_parser.rb'
class OracleLinuxOvalImporter
attr_accessor :url, :filepath
def initialize
@url = 'https://linux.oracle.com/security/oval/com.oracle.elsa-all.xml.bz2'
@filepath = '/data_importer/data/oracle_oval.xml.bz2'
end
def get_file
if File.exist? filepath
puts "#{filepath} exists"
else
`wget -O #{filepath} #{url}`
end
end
def decompress_bz2
Bzip2::FFI::Reader.read(filepath)
end
def xml_doc
get_file
xml = decompress_bz2
OvalParser.new(xml).doc
end
end

View file

@ -19,7 +19,7 @@ class JsonHelper
end end
def self.read_json_from_file(filename) def self.read_json_from_file(filename)
JSON.parse(File.read(filename, :encoding => 'utf-8'), symbolize_names: true) JSON.parse(File.read(filename), symbolize_names: true)
end end
def self.read_json_from_url(url) def self.read_json_from_url(url)

View file

@ -1,54 +0,0 @@
require 'nokogiri'
class OvalParser
attr_accessor :data, :doc, :root
def initialize(data)
@data = data
@doc = Nokogiri::XML(data)
@root = doc.root
end
def get_definitions
root.xpath("//xmlns:definition")
end
def get_def_ids
definitions = get_definitions
definitions.map { |d| d.xpath(".//@id").text }
end
def oval_defs_for_id(id)
definitions = get_definitions
definitions.select do |definition|
definition if definition.attributes.dig('id').value == id
end
end
def get_cve_def_info
# largerly used this code here as a guide:
# https://github.com/OpenSCAP/openscap-daemon/blob/1b9e9d4849573e1ce09728cc61c4564e5d605a8e/openscap_daemon/cve_scanner/generate_summary.py#L83-L104
def_ids = get_def_ids
def_ids.map do |id|
oval_defs = oval_defs_for_id(id)
oval_defs.map do |oval_def|
title = oval_def.xpath(".//xmlns:title").text
cve_meta = oval_def.xpath(".//xmlns:reference").select { |n| n.attributes.dig('source').value == 'CVE' }
cve_ids = cve_meta.map { |cve_m| cve_m.attributes.dig('ref_id').value || 'None' }
cve_urls = cve_meta.map { |cve_m| cve_m.attributes.dig('ref_url').value || 'None' }
description = oval_def.xpath(".//xmlns:description").text
severity = oval_def.xpath(".//xmlns:severity").text
my_sample_data_hash = {
:title => title,
:cve_ids => cve_ids,
:cve_urls => cve_urls,
:description => description,
:severity => severity
}
binding.pry
my_sample_data_hash
{}
end
end
end
end