Merge branch 'convert_pastebinner_to_rails_app' into 'master'
Convert pastebinner to rails app See merge request bpmcdevitt/pastebinner!1
This commit is contained in:
commit
8dff02ca42
135 changed files with 9835 additions and 103 deletions
1
.browserslistrc
Normal file
1
.browserslistrc
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
defaults
|
||||||
3
.env
Normal file
3
.env
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
REDIS_SERVER_URL=redis://pastebinner-redis:6379/0
|
||||||
|
REDIS_SERVER_URL_PASTES_HOSTS=pastebinner-redis
|
||||||
|
REDIS_SERVER_URL_PASTES_DB=0
|
||||||
52
.gitignore
vendored
52
.gitignore
vendored
|
|
@ -1,16 +1,38 @@
|
||||||
/.bundle/
|
# See https://help.github.com/articles/ignoring-files for more about ignoring files.
|
||||||
/.yardoc
|
#
|
||||||
/_yardoc/
|
# If you find yourself ignoring temporary files generated by your text editor
|
||||||
/coverage/
|
# or operating system, you probably want to add a global ignore instead:
|
||||||
/doc/
|
# git config --global core.excludesfile '~/.gitignore_global'
|
||||||
/pkg/
|
|
||||||
/spec/reports/
|
|
||||||
/tmp/
|
|
||||||
/data
|
|
||||||
/vendor
|
|
||||||
Gemfile.lock
|
|
||||||
.config
|
|
||||||
.env
|
|
||||||
|
|
||||||
# rspec failure tracking
|
# Ignore bundler config.
|
||||||
.rspec_status
|
/.bundle
|
||||||
|
|
||||||
|
# Ignore the default SQLite database.
|
||||||
|
/db/*.sqlite3
|
||||||
|
/db/*.sqlite3-journal
|
||||||
|
|
||||||
|
# Ignore all logfiles and tempfiles.
|
||||||
|
/log/*
|
||||||
|
/tmp/*
|
||||||
|
!/log/.keep
|
||||||
|
!/tmp/.keep
|
||||||
|
|
||||||
|
# Ignore uploaded files in development.
|
||||||
|
/storage/*
|
||||||
|
!/storage/.keep
|
||||||
|
|
||||||
|
/public/assets
|
||||||
|
.byebug_history
|
||||||
|
|
||||||
|
# Ignore master key for decrypting credentials and more.
|
||||||
|
./config/master.key
|
||||||
|
./config/local_env.yml
|
||||||
|
.pastebin_creds
|
||||||
|
/public/packs
|
||||||
|
/public/packs-test
|
||||||
|
/node_modules
|
||||||
|
/yarn-error.log
|
||||||
|
yarn-debug.log*
|
||||||
|
.yarn-integrity
|
||||||
|
pastebinner.pid
|
||||||
|
pids/*
|
||||||
|
|
|
||||||
1
.ruby-version
Normal file
1
.ruby-version
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
2.6.0
|
||||||
22
Dockerfile
22
Dockerfile
|
|
@ -1,13 +1,17 @@
|
||||||
FROM ruby:2.5
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM ruby:2.6
|
||||||
|
RUN apt-get update -qq && apt-get install -y npm nodejs postgresql-client net-tools iproute2 iputils-ping
|
||||||
RUN mkdir /usr/src/app
|
RUN mkdir /usr/src/app
|
||||||
ADD . /usr/src/app/
|
ADD . /usr/src/app
|
||||||
WORKDIR /usr/src/app/
|
WORKDIR /usr/src/app
|
||||||
|
|
||||||
RUN gem update --system
|
|
||||||
RUN gem install bundler
|
RUN gem install bundler
|
||||||
|
|
||||||
RUN bundle install
|
RUN bundle install
|
||||||
|
RUN npm install -g yarn
|
||||||
|
RUN yarn install --check-files
|
||||||
|
|
||||||
WORKDIR /usr/src/app/bin
|
# expose the rails port
|
||||||
CMD ["./console"]
|
EXPOSE 3000
|
||||||
|
|
||||||
|
# Configure the main process to run when running the image
|
||||||
|
CMD ["rails", "server", "-b", "0.0.0.0"]
|
||||||
|
#CMD ["rails", "console"]
|
||||||
|
|
|
||||||
61
Gemfile
61
Gemfile
|
|
@ -1,6 +1,61 @@
|
||||||
source 'https://rubygems.org'
|
source 'https://rubygems.org'
|
||||||
|
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
|
||||||
|
|
||||||
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
ruby '2.6.9'
|
||||||
|
|
||||||
# Specify your gem's dependencies in pastebinner.gemspec
|
# Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
|
||||||
gemspec
|
gem 'rails', '~> 6.0.0'
|
||||||
|
# Use sqlite3 as the database for Active Record
|
||||||
|
gem 'sqlite3', '~> 1.4'
|
||||||
|
# Use Puma as the app server
|
||||||
|
gem 'puma', '~> 3.11'
|
||||||
|
# Use SCSS for stylesheets
|
||||||
|
gem 'sass-rails', '~> 5'
|
||||||
|
# Transpile app-like JavaScript. Read more: https://github.com/rails/webpacker
|
||||||
|
gem 'webpacker', '~> 4.0'
|
||||||
|
# Turbolinks makes navigating your web application faster. Read more: https://github.com/turbolinks/turbolinks
|
||||||
|
gem 'turbolinks', '~> 5'
|
||||||
|
# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
|
||||||
|
gem 'jbuilder', '~> 2.7'
|
||||||
|
# Use Redis adapter to run Action Cable in production
|
||||||
|
gem 'redis', '~> 4.0'
|
||||||
|
# Use Active Model has_secure_password
|
||||||
|
# gem 'bcrypt', '~> 3.1.7'
|
||||||
|
gem 'rest-client', '~> 2.0'
|
||||||
|
gem 'elasticsearch-model', '~> 2.0'
|
||||||
|
gem 'elasticsearch-rails', '~> 2.0'
|
||||||
|
gem 'sidekiq', '~> 5.2.5'
|
||||||
|
gem 'sidekiq-scheduler'
|
||||||
|
gem 'pry'
|
||||||
|
gem "sidekiq-cron", "~> 1.1"
|
||||||
|
# Use Active Storage variant
|
||||||
|
# gem 'image_processing', '~> 1.2'
|
||||||
|
|
||||||
|
# Reduces boot times through caching; required in config/boot.rb
|
||||||
|
gem 'bootsnap', '>= 1.4.2', require: false
|
||||||
|
|
||||||
|
group :development, :test do
|
||||||
|
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
|
||||||
|
gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
|
||||||
|
end
|
||||||
|
|
||||||
|
group :development do
|
||||||
|
# Access an interactive console on exception pages or by calling 'console' anywhere in the code.
|
||||||
|
gem 'web-console', '>= 3.3.0'
|
||||||
|
gem 'listen', '>= 3.0.5', '< 3.2'
|
||||||
|
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
|
||||||
|
gem 'spring'
|
||||||
|
gem 'spring-watcher-listen', '~> 2.0.0'
|
||||||
|
gem 'pry-rails'
|
||||||
|
end
|
||||||
|
|
||||||
|
group :test do
|
||||||
|
# Adds support for Capybara system testing and selenium driver
|
||||||
|
gem 'capybara', '>= 2.15'
|
||||||
|
gem 'selenium-webdriver'
|
||||||
|
# Easy installation and use of web drivers to run system tests with browsers
|
||||||
|
gem 'webdrivers'
|
||||||
|
end
|
||||||
|
|
||||||
|
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
|
||||||
|
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
|
||||||
|
|
|
||||||
323
Gemfile.lock
Normal file
323
Gemfile.lock
Normal file
|
|
@ -0,0 +1,323 @@
|
||||||
|
GEM
|
||||||
|
remote: https://rubygems.org/
|
||||||
|
specs:
|
||||||
|
actioncable (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
nio4r (~> 2.0)
|
||||||
|
websocket-driver (>= 0.6.1)
|
||||||
|
actionmailbox (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
activejob (= 6.0.4)
|
||||||
|
activerecord (= 6.0.4)
|
||||||
|
activestorage (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
mail (>= 2.7.1)
|
||||||
|
actionmailer (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
actionview (= 6.0.4)
|
||||||
|
activejob (= 6.0.4)
|
||||||
|
mail (~> 2.5, >= 2.5.4)
|
||||||
|
rails-dom-testing (~> 2.0)
|
||||||
|
actionpack (6.0.4)
|
||||||
|
actionview (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
rack (~> 2.0, >= 2.0.8)
|
||||||
|
rack-test (>= 0.6.3)
|
||||||
|
rails-dom-testing (~> 2.0)
|
||||||
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
||||||
|
actiontext (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
activerecord (= 6.0.4)
|
||||||
|
activestorage (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
nokogiri (>= 1.8.5)
|
||||||
|
actionview (6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
builder (~> 3.1)
|
||||||
|
erubi (~> 1.4)
|
||||||
|
rails-dom-testing (~> 2.0)
|
||||||
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
||||||
|
activejob (6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
globalid (>= 0.3.6)
|
||||||
|
activemodel (6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
activerecord (6.0.4)
|
||||||
|
activemodel (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
activestorage (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
activejob (= 6.0.4)
|
||||||
|
activerecord (= 6.0.4)
|
||||||
|
marcel (~> 1.0.0)
|
||||||
|
activesupport (6.0.4)
|
||||||
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||||
|
i18n (>= 0.7, < 2)
|
||||||
|
minitest (~> 5.1)
|
||||||
|
tzinfo (~> 1.1)
|
||||||
|
zeitwerk (~> 2.2, >= 2.2.2)
|
||||||
|
addressable (2.8.0)
|
||||||
|
public_suffix (>= 2.0.2, < 5.0)
|
||||||
|
bindex (0.8.1)
|
||||||
|
bootsnap (1.7.7)
|
||||||
|
msgpack (~> 1.0)
|
||||||
|
builder (3.2.4)
|
||||||
|
byebug (11.1.3)
|
||||||
|
capybara (3.35.3)
|
||||||
|
addressable
|
||||||
|
mini_mime (>= 0.1.3)
|
||||||
|
nokogiri (~> 1.8)
|
||||||
|
rack (>= 1.6.0)
|
||||||
|
rack-test (>= 0.6.3)
|
||||||
|
regexp_parser (>= 1.5, < 3.0)
|
||||||
|
xpath (~> 3.2)
|
||||||
|
childprocess (3.0.0)
|
||||||
|
coderay (1.1.3)
|
||||||
|
concurrent-ruby (1.1.9)
|
||||||
|
connection_pool (2.2.5)
|
||||||
|
crass (1.0.6)
|
||||||
|
domain_name (0.5.20190701)
|
||||||
|
unf (>= 0.0.5, < 1.0.0)
|
||||||
|
e2mmap (0.1.0)
|
||||||
|
elasticsearch (2.0.2)
|
||||||
|
elasticsearch-api (= 2.0.2)
|
||||||
|
elasticsearch-transport (= 2.0.2)
|
||||||
|
elasticsearch-api (2.0.2)
|
||||||
|
multi_json
|
||||||
|
elasticsearch-model (2.0.1)
|
||||||
|
activesupport (> 3)
|
||||||
|
elasticsearch (~> 2)
|
||||||
|
hashie
|
||||||
|
elasticsearch-rails (2.0.1)
|
||||||
|
elasticsearch-transport (2.0.2)
|
||||||
|
faraday
|
||||||
|
multi_json
|
||||||
|
erubi (1.10.0)
|
||||||
|
et-orbi (1.2.4)
|
||||||
|
tzinfo
|
||||||
|
faraday (1.7.0)
|
||||||
|
faraday-em_http (~> 1.0)
|
||||||
|
faraday-em_synchrony (~> 1.0)
|
||||||
|
faraday-excon (~> 1.1)
|
||||||
|
faraday-httpclient (~> 1.0.1)
|
||||||
|
faraday-net_http (~> 1.0)
|
||||||
|
faraday-net_http_persistent (~> 1.1)
|
||||||
|
faraday-patron (~> 1.0)
|
||||||
|
faraday-rack (~> 1.0)
|
||||||
|
multipart-post (>= 1.2, < 3)
|
||||||
|
ruby2_keywords (>= 0.0.4)
|
||||||
|
faraday-em_http (1.0.0)
|
||||||
|
faraday-em_synchrony (1.0.0)
|
||||||
|
faraday-excon (1.1.0)
|
||||||
|
faraday-httpclient (1.0.1)
|
||||||
|
faraday-net_http (1.0.1)
|
||||||
|
faraday-net_http_persistent (1.2.0)
|
||||||
|
faraday-patron (1.0.0)
|
||||||
|
faraday-rack (1.0.0)
|
||||||
|
ffi (1.15.3)
|
||||||
|
fugit (1.5.0)
|
||||||
|
et-orbi (~> 1.1, >= 1.1.8)
|
||||||
|
raabro (~> 1.4)
|
||||||
|
globalid (0.5.2)
|
||||||
|
activesupport (>= 5.0)
|
||||||
|
hashie (4.1.0)
|
||||||
|
http-accept (1.7.0)
|
||||||
|
http-cookie (1.0.4)
|
||||||
|
domain_name (~> 0.5)
|
||||||
|
i18n (1.8.10)
|
||||||
|
concurrent-ruby (~> 1.0)
|
||||||
|
jbuilder (2.11.2)
|
||||||
|
activesupport (>= 5.0.0)
|
||||||
|
listen (3.1.5)
|
||||||
|
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||||
|
rb-inotify (~> 0.9, >= 0.9.7)
|
||||||
|
ruby_dep (~> 1.2)
|
||||||
|
loofah (2.11.0)
|
||||||
|
crass (~> 1.0.2)
|
||||||
|
nokogiri (>= 1.5.9)
|
||||||
|
mail (2.7.1)
|
||||||
|
mini_mime (>= 0.1.1)
|
||||||
|
marcel (1.0.1)
|
||||||
|
method_source (1.0.0)
|
||||||
|
mime-types (3.3.1)
|
||||||
|
mime-types-data (~> 3.2015)
|
||||||
|
mime-types-data (3.2021.0704)
|
||||||
|
mini_mime (1.1.0)
|
||||||
|
mini_portile2 (2.6.1)
|
||||||
|
minitest (5.14.4)
|
||||||
|
msgpack (1.4.2)
|
||||||
|
multi_json (1.15.0)
|
||||||
|
multipart-post (2.1.1)
|
||||||
|
netrc (0.11.0)
|
||||||
|
nio4r (2.5.8)
|
||||||
|
nokogiri (1.12.2)
|
||||||
|
mini_portile2 (~> 2.6.1)
|
||||||
|
racc (~> 1.4)
|
||||||
|
pry (0.14.1)
|
||||||
|
coderay (~> 1.1)
|
||||||
|
method_source (~> 1.0)
|
||||||
|
pry-rails (0.3.9)
|
||||||
|
pry (>= 0.10.4)
|
||||||
|
public_suffix (4.0.6)
|
||||||
|
puma (3.12.6)
|
||||||
|
raabro (1.4.0)
|
||||||
|
racc (1.5.2)
|
||||||
|
rack (2.2.3)
|
||||||
|
rack-protection (2.1.0)
|
||||||
|
rack
|
||||||
|
rack-proxy (0.7.0)
|
||||||
|
rack
|
||||||
|
rack-test (1.1.0)
|
||||||
|
rack (>= 1.0, < 3)
|
||||||
|
rails (6.0.4)
|
||||||
|
actioncable (= 6.0.4)
|
||||||
|
actionmailbox (= 6.0.4)
|
||||||
|
actionmailer (= 6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
actiontext (= 6.0.4)
|
||||||
|
actionview (= 6.0.4)
|
||||||
|
activejob (= 6.0.4)
|
||||||
|
activemodel (= 6.0.4)
|
||||||
|
activerecord (= 6.0.4)
|
||||||
|
activestorage (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
bundler (>= 1.3.0)
|
||||||
|
railties (= 6.0.4)
|
||||||
|
sprockets-rails (>= 2.0.0)
|
||||||
|
rails-dom-testing (2.0.3)
|
||||||
|
activesupport (>= 4.2.0)
|
||||||
|
nokogiri (>= 1.6)
|
||||||
|
rails-html-sanitizer (1.3.0)
|
||||||
|
loofah (~> 2.3)
|
||||||
|
railties (6.0.4)
|
||||||
|
actionpack (= 6.0.4)
|
||||||
|
activesupport (= 6.0.4)
|
||||||
|
method_source
|
||||||
|
rake (>= 0.8.7)
|
||||||
|
thor (>= 0.20.3, < 2.0)
|
||||||
|
rake (13.0.6)
|
||||||
|
rb-fsevent (0.11.0)
|
||||||
|
rb-inotify (0.10.1)
|
||||||
|
ffi (~> 1.0)
|
||||||
|
redis (4.1.4)
|
||||||
|
regexp_parser (2.1.1)
|
||||||
|
rest-client (2.1.0)
|
||||||
|
http-accept (>= 1.7.0, < 2.0)
|
||||||
|
http-cookie (>= 1.0.2, < 2.0)
|
||||||
|
mime-types (>= 1.16, < 4.0)
|
||||||
|
netrc (~> 0.8)
|
||||||
|
ruby2_keywords (0.0.5)
|
||||||
|
ruby_dep (1.5.0)
|
||||||
|
rubyzip (2.3.2)
|
||||||
|
rufus-scheduler (3.8.0)
|
||||||
|
fugit (~> 1.1, >= 1.1.6)
|
||||||
|
sass (3.7.4)
|
||||||
|
sass-listen (~> 4.0.0)
|
||||||
|
sass-listen (4.0.0)
|
||||||
|
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||||
|
rb-inotify (~> 0.9, >= 0.9.7)
|
||||||
|
sass-rails (5.1.0)
|
||||||
|
railties (>= 5.2.0)
|
||||||
|
sass (~> 3.1)
|
||||||
|
sprockets (>= 2.8, < 4.0)
|
||||||
|
sprockets-rails (>= 2.0, < 4.0)
|
||||||
|
tilt (>= 1.1, < 3)
|
||||||
|
selenium-webdriver (3.142.7)
|
||||||
|
childprocess (>= 0.5, < 4.0)
|
||||||
|
rubyzip (>= 1.2.2)
|
||||||
|
sidekiq (5.2.9)
|
||||||
|
connection_pool (~> 2.2, >= 2.2.2)
|
||||||
|
rack (~> 2.0)
|
||||||
|
rack-protection (>= 1.5.0)
|
||||||
|
redis (>= 3.3.5, < 4.2)
|
||||||
|
sidekiq-cron (1.2.0)
|
||||||
|
fugit (~> 1.1)
|
||||||
|
sidekiq (>= 4.2.1)
|
||||||
|
sidekiq-scheduler (3.1.0)
|
||||||
|
e2mmap
|
||||||
|
redis (>= 3, < 5)
|
||||||
|
rufus-scheduler (~> 3.2)
|
||||||
|
sidekiq (>= 3)
|
||||||
|
thwait
|
||||||
|
tilt (>= 1.4.0)
|
||||||
|
spring (2.1.1)
|
||||||
|
spring-watcher-listen (2.0.1)
|
||||||
|
listen (>= 2.7, < 4.0)
|
||||||
|
spring (>= 1.2, < 3.0)
|
||||||
|
sprockets (3.7.2)
|
||||||
|
concurrent-ruby (~> 1.0)
|
||||||
|
rack (> 1, < 3)
|
||||||
|
sprockets-rails (3.2.2)
|
||||||
|
actionpack (>= 4.0)
|
||||||
|
activesupport (>= 4.0)
|
||||||
|
sprockets (>= 3.0.0)
|
||||||
|
sqlite3 (1.4.2)
|
||||||
|
thor (1.1.0)
|
||||||
|
thread_safe (0.3.6)
|
||||||
|
thwait (0.2.0)
|
||||||
|
e2mmap
|
||||||
|
tilt (2.0.10)
|
||||||
|
turbolinks (5.2.1)
|
||||||
|
turbolinks-source (~> 5.2)
|
||||||
|
turbolinks-source (5.2.0)
|
||||||
|
tzinfo (1.2.9)
|
||||||
|
thread_safe (~> 0.1)
|
||||||
|
unf (0.1.4)
|
||||||
|
unf_ext
|
||||||
|
unf_ext (0.0.7.7)
|
||||||
|
web-console (4.1.0)
|
||||||
|
actionview (>= 6.0.0)
|
||||||
|
activemodel (>= 6.0.0)
|
||||||
|
bindex (>= 0.4.0)
|
||||||
|
railties (>= 6.0.0)
|
||||||
|
webdrivers (4.6.0)
|
||||||
|
nokogiri (~> 1.6)
|
||||||
|
rubyzip (>= 1.3.0)
|
||||||
|
selenium-webdriver (>= 3.0, < 4.0)
|
||||||
|
webpacker (4.3.0)
|
||||||
|
activesupport (>= 4.2)
|
||||||
|
rack-proxy (>= 0.6.1)
|
||||||
|
railties (>= 4.2)
|
||||||
|
websocket-driver (0.7.5)
|
||||||
|
websocket-extensions (>= 0.1.0)
|
||||||
|
websocket-extensions (0.1.5)
|
||||||
|
xpath (3.2.0)
|
||||||
|
nokogiri (~> 1.8)
|
||||||
|
zeitwerk (2.4.2)
|
||||||
|
|
||||||
|
PLATFORMS
|
||||||
|
ruby
|
||||||
|
|
||||||
|
DEPENDENCIES
|
||||||
|
bootsnap (>= 1.4.2)
|
||||||
|
byebug
|
||||||
|
capybara (>= 2.15)
|
||||||
|
elasticsearch-model (~> 2.0)
|
||||||
|
elasticsearch-rails (~> 2.0)
|
||||||
|
jbuilder (~> 2.7)
|
||||||
|
listen (>= 3.0.5, < 3.2)
|
||||||
|
pry
|
||||||
|
pry-rails
|
||||||
|
puma (~> 3.11)
|
||||||
|
rails (~> 6.0.0)
|
||||||
|
redis (~> 4.0)
|
||||||
|
rest-client (~> 2.0)
|
||||||
|
sass-rails (~> 5)
|
||||||
|
selenium-webdriver
|
||||||
|
sidekiq (~> 5.2.5)
|
||||||
|
sidekiq-cron (~> 1.1)
|
||||||
|
sidekiq-scheduler
|
||||||
|
spring
|
||||||
|
spring-watcher-listen (~> 2.0.0)
|
||||||
|
sqlite3 (~> 1.4)
|
||||||
|
turbolinks (~> 5)
|
||||||
|
tzinfo-data
|
||||||
|
web-console (>= 3.3.0)
|
||||||
|
webdrivers
|
||||||
|
webpacker (~> 4.0)
|
||||||
|
|
||||||
|
RUBY VERSION
|
||||||
|
ruby 2.6.0p0
|
||||||
|
|
||||||
|
BUNDLED WITH
|
||||||
|
1.17.2
|
||||||
19
NOTES.md
Normal file
19
NOTES.md
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
#### DEPLOYING TO SERVER
|
||||||
|
# I USED OVH VPS WITH ARCHLINUX
|
||||||
|
# run as sudo user
|
||||||
|
|
||||||
|
# step 1: get rbenv dependencies & other useful programs
|
||||||
|
sudo pacman -S --needed base-devel libffi libyaml openssl zlib go wget curl vim
|
||||||
|
|
||||||
|
# step 2: install aur helper
|
||||||
|
wget https://aur.archlinux.org/cgit/aur.git/snapshot/yay.tar.gz && tar xzvf yay.tar.gz
|
||||||
|
pacman -S ca-certificates-utils
|
||||||
|
cd yay && makepkg -Acs
|
||||||
|
sudo pacman -U pacman -S ca-certificates-utils
|
||||||
|
|
||||||
|
# get rbenv and ruby-build
|
||||||
|
yay -S rbenv
|
||||||
|
yay -S ruby-build
|
||||||
|
|
||||||
|
# install a version of ruby
|
||||||
|
rbenv install 2.6.0
|
||||||
84
README.md
84
README.md
|
|
@ -1,23 +1,11 @@
|
||||||
# Pastebinner
|
# README
|
||||||
Check out the examples folder for some examples. I will add more soon.
|
You need docker-compose installed.
|
||||||
|
- disable ipv6 if you dont want to use it in this file under the networks: section. You can just comment the ipv6 subnet,gateway,enable lines to do that
|
||||||
|
|
||||||
## Configuration
|
### Before you start:
|
||||||
|
You should have a Pastebin Pro API membership. You will also need to whitelist your IP Address. I have had success with both Ipv4 and Ipv6 addresses. This allows you access to the scraping API: https://pastebin.com/doc_scraping_api
|
||||||
|
|
||||||
#### Docker
|
1. Create a .pastebin_creds file that contains the following environment vars
|
||||||
- Build the image:
|
|
||||||
```shell
|
|
||||||
docker build -t pastebinner .
|
|
||||||
```
|
|
||||||
- Run the image:
|
|
||||||
```shell
|
|
||||||
docker run -it --env-file .env --network="host" pastebinner
|
|
||||||
```
|
|
||||||
#### Console
|
|
||||||
|
|
||||||
The console will launch you into a running pry session where you will have access to an elasticsearch object and a pastebinner object. From there you can poke around and investigate further how the program works.
|
|
||||||
|
|
||||||
I am working on setting it up with docker-compose to connect elasticsearch and redis in docker containers.
|
|
||||||
Set the following environment variables in a file named .env and source them:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
pastebin_api_key
|
pastebin_api_key
|
||||||
|
|
@ -25,53 +13,19 @@ pastebin_username
|
||||||
pastebin_password
|
pastebin_password
|
||||||
```
|
```
|
||||||
|
|
||||||
optional environment variables if planning on using elasticsearch and redis
|
this should store the creds in a file that is .gitignored and will allow the application to correctly scrape paste data.
|
||||||
```
|
|
||||||
elasticsearch_url
|
|
||||||
redis_url
|
|
||||||
```
|
|
||||||
|
|
||||||
|
### To use:
|
||||||
|
`docker-compose up`
|
||||||
|
|
||||||
If you want to scrape pastes, you can view a json response of the latest pastes by using the ```-s``` or ```--scrape_public``` options.
|
This will create the following containers and services:
|
||||||
Creating pastes is built in, check ```lib/api_client```
|
- pastebinner-rails
|
||||||
|
- pastebinner-elasticsearch
|
||||||
|
- pastebinner-redis
|
||||||
|
- pastebinner-kibana
|
||||||
|
- pastebinner-sidekiq
|
||||||
|
|
||||||
#### Commandline
|
### Interacting:
|
||||||
The command line app can be used as follows:
|
You can access the Kibana search interface at https://localhost:5601. This is just an interface into Kibana. You will need to create the `pastes` index pattern at first visit. It should then be scraping public pastes every 1 min. Any duplicate pastes keys are stored in Redis and will not be retrieved twice so we are not sending dupes to our ES db.
|
||||||
|
To view status of jobs you can visit the sidekiq dashboard at http://localhost:3000/sidekiq
|
||||||
```shell
|
To view the status of the worker job, you can view the sidekiq logs with `docker-compose logs pastebinner-sidekiq`
|
||||||
Usage: pastebinner [options]
|
|
||||||
-h, --help Show this help messae
|
|
||||||
-v, --verbose Verbose http output (WIP)
|
|
||||||
-s, --scrape_public Scrape public pastes
|
|
||||||
-r, --raw Raw paste. Requires --key passed with a valid key
|
|
||||||
-g, --get_keys Get unique paste keys from public pastes
|
|
||||||
-k, --key= Unique paste key
|
|
||||||
-d, --download Download all public pastes to data directory
|
|
||||||
-j, --json Download all public pastes as a json into data directory
|
|
||||||
-t, --trending Trending pastes
|
|
||||||
```
|
|
||||||
### Downloading pastes
|
|
||||||
Create a directory in the git repo named data, this is where the raw pastes will be stored.
|
|
||||||
Here is an example of what it looks like:
|
|
||||||
|
|
||||||
##### Raw pastes to files
|
|
||||||
```shell
|
|
||||||
pastebinner --download
|
|
||||||
Downloading paste data into the data directory...
|
|
||||||
Complete.
|
|
||||||
```
|
|
||||||
|
|
||||||
##### JSON file with paste_metadata and paste_text
|
|
||||||
```shell
|
|
||||||
pastebinner --json
|
|
||||||
Downloading paste data as a json into the data directory...
|
|
||||||
Complete.
|
|
||||||
```
|
|
||||||
The data directory will then be populated with pastebin raw paste files with the following naming scheme:
|
|
||||||
|
|
||||||
```pastebin_paste_key_agiArDuG.raw``` or ```pastebin_paste_key_agiArDuG.json```
|
|
||||||
### To Add:
|
|
||||||
- fulltext search of raw pastes via elastic search, mysql, postgres, sqlite, or some other method
|
|
||||||
- adding exceptions
|
|
||||||
- adding configuration file support
|
|
||||||
- adding rspec tests
|
|
||||||
|
|
|
||||||
8
Rakefile
8
Rakefile
|
|
@ -1,6 +1,6 @@
|
||||||
require 'bundler/gem_tasks'
|
# Add your own tasks in files placed in lib/tasks ending in .rake,
|
||||||
require 'rspec/core/rake_task'
|
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
|
||||||
|
|
||||||
RSpec::Core::RakeTask.new(:spec)
|
require_relative 'config/application'
|
||||||
|
|
||||||
task default: :spec
|
Rails.application.load_tasks
|
||||||
|
|
|
||||||
2
app/assets/config/manifest.js
Normal file
2
app/assets/config/manifest.js
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
//= link_tree ../images
|
||||||
|
//= link_directory ../stylesheets .css
|
||||||
0
app/assets/images/.keep
Normal file
0
app/assets/images/.keep
Normal file
15
app/assets/stylesheets/application.css
Normal file
15
app/assets/stylesheets/application.css
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
/*
|
||||||
|
* This is a manifest file that'll be compiled into application.css, which will include all the files
|
||||||
|
* listed below.
|
||||||
|
*
|
||||||
|
* Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's
|
||||||
|
* vendor/assets/stylesheets directory can be referenced here using a relative path.
|
||||||
|
*
|
||||||
|
* You're free to add application-wide styles to this file and they'll appear at the bottom of the
|
||||||
|
* compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
|
||||||
|
* files in this directory. Styles in this file should be added after the last require_* statement.
|
||||||
|
* It is generally better to create a new file per style scope.
|
||||||
|
*
|
||||||
|
*= require_tree .
|
||||||
|
*= require_self
|
||||||
|
*/
|
||||||
3
app/assets/stylesheets/pastes.scss
Normal file
3
app/assets/stylesheets/pastes.scss
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
// Place all the styles related to the Pastes controller here.
|
||||||
|
// They will automatically be included in application.css.
|
||||||
|
// You can use Sass (SCSS) here: http://sass-lang.com/
|
||||||
84
app/assets/stylesheets/scaffolds.scss
Normal file
84
app/assets/stylesheets/scaffolds.scss
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
body {
|
||||||
|
background-color: #fff;
|
||||||
|
color: #333;
|
||||||
|
margin: 33px;
|
||||||
|
font-family: verdana, arial, helvetica, sans-serif;
|
||||||
|
font-size: 13px;
|
||||||
|
line-height: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
p, ol, ul, td {
|
||||||
|
font-family: verdana, arial, helvetica, sans-serif;
|
||||||
|
font-size: 13px;
|
||||||
|
line-height: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre {
|
||||||
|
background-color: #eee;
|
||||||
|
padding: 10px;
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
color: #000;
|
||||||
|
|
||||||
|
&:visited {
|
||||||
|
color: #666;
|
||||||
|
}
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
color: #fff;
|
||||||
|
background-color: #000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
th {
|
||||||
|
padding-bottom: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
td {
|
||||||
|
padding: 0 5px 7px;
|
||||||
|
}
|
||||||
|
|
||||||
|
div {
|
||||||
|
&.field, &.actions {
|
||||||
|
margin-bottom: 10px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#notice {
|
||||||
|
color: green;
|
||||||
|
}
|
||||||
|
|
||||||
|
.field_with_errors {
|
||||||
|
padding: 2px;
|
||||||
|
background-color: red;
|
||||||
|
display: table;
|
||||||
|
}
|
||||||
|
|
||||||
|
#error_explanation {
|
||||||
|
width: 450px;
|
||||||
|
border: 2px solid red;
|
||||||
|
padding: 7px 7px 0;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
background-color: #f0f0f0;
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
text-align: left;
|
||||||
|
font-weight: bold;
|
||||||
|
padding: 5px 5px 5px 15px;
|
||||||
|
font-size: 12px;
|
||||||
|
margin: -7px -7px 0;
|
||||||
|
background-color: #c00;
|
||||||
|
color: #fff;
|
||||||
|
}
|
||||||
|
|
||||||
|
ul li {
|
||||||
|
font-size: 12px;
|
||||||
|
list-style: square;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
label {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
9
app/assets/stylesheets/search.scss
Normal file
9
app/assets/stylesheets/search.scss
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
// Place all the styles related to the Search controller here.
|
||||||
|
// They will automatically be included in application.css.
|
||||||
|
// You can use Sass (SCSS) here: http://sass-lang.com/
|
||||||
|
|
||||||
|
.search_results em {
|
||||||
|
background-color: yellow;
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
4
app/channels/application_cable/channel.rb
Normal file
4
app/channels/application_cable/channel.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
module ApplicationCable
|
||||||
|
class Channel < ActionCable::Channel::Base
|
||||||
|
end
|
||||||
|
end
|
||||||
4
app/channels/application_cable/connection.rb
Normal file
4
app/channels/application_cable/connection.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
module ApplicationCable
|
||||||
|
class Connection < ActionCable::Connection::Base
|
||||||
|
end
|
||||||
|
end
|
||||||
2
app/controllers/application_controller.rb
Normal file
2
app/controllers/application_controller.rb
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
class ApplicationController < ActionController::Base
|
||||||
|
end
|
||||||
0
app/controllers/concerns/.keep
Normal file
0
app/controllers/concerns/.keep
Normal file
74
app/controllers/pastes_controller.rb
Normal file
74
app/controllers/pastes_controller.rb
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
class PastesController < ApplicationController
|
||||||
|
before_action :set_paste, only: [:show, :edit, :update, :destroy]
|
||||||
|
|
||||||
|
# GET /pastes
|
||||||
|
# GET /pastes.json
|
||||||
|
def index
|
||||||
|
@pastes = Paste.all
|
||||||
|
end
|
||||||
|
|
||||||
|
# GET /pastes/1
|
||||||
|
# GET /pastes/1.json
|
||||||
|
def show
|
||||||
|
end
|
||||||
|
|
||||||
|
# GET /pastes/new
|
||||||
|
# def new
|
||||||
|
# @paste = Paste.new
|
||||||
|
# end
|
||||||
|
|
||||||
|
# # GET /pastes/1/edit
|
||||||
|
# def edit
|
||||||
|
# end
|
||||||
|
|
||||||
|
# POST /pastes
|
||||||
|
# POST /pastes.json
|
||||||
|
# def create
|
||||||
|
# @paste = Paste.new(paste_params)
|
||||||
|
#
|
||||||
|
# respond_to do |format|
|
||||||
|
# if @paste.save
|
||||||
|
# format.html { redirect_to @paste, notice: 'Paste was successfully created.' }
|
||||||
|
# format.json { render :show, status: :created, location: @paste }
|
||||||
|
# else
|
||||||
|
# format.html { render :new }
|
||||||
|
# format.json { render json: @paste.errors, status: :unprocessable_entity }
|
||||||
|
# end
|
||||||
|
# end
|
||||||
|
# end
|
||||||
|
|
||||||
|
# PATCH/PUT /pastes/1
|
||||||
|
# PATCH/PUT /pastes/1.json
|
||||||
|
# def update
|
||||||
|
# respond_to do |format|
|
||||||
|
# if @paste.update(paste_params)
|
||||||
|
# format.html { redirect_to @paste, notice: 'Paste was successfully updated.' }
|
||||||
|
# format.json { render :show, status: :ok, location: @paste }
|
||||||
|
# else
|
||||||
|
# format.html { render :edit }
|
||||||
|
# format.json { render json: @paste.errors, status: :unprocessable_entity }
|
||||||
|
# end
|
||||||
|
# end
|
||||||
|
# end
|
||||||
|
|
||||||
|
# DELETE /pastes/1
|
||||||
|
# DELETE /pastes/1.json
|
||||||
|
# def destroy
|
||||||
|
# @paste.destroy
|
||||||
|
# respond_to do |format|
|
||||||
|
# format.html { redirect_to pastes_url, notice: 'Paste was successfully destroyed.' }
|
||||||
|
# format.json { head :no_content }
|
||||||
|
# end
|
||||||
|
# end
|
||||||
|
|
||||||
|
private
|
||||||
|
# Use callbacks to share common setup or constraints between actions.
|
||||||
|
# def set_paste
|
||||||
|
# @paste = Paste.find(params[:id])
|
||||||
|
# end
|
||||||
|
|
||||||
|
# Never trust parameters from the scary internet, only allow the white list through.
|
||||||
|
# def paste_params
|
||||||
|
# params.require(:paste).permit(:search_pastes)
|
||||||
|
# end
|
||||||
|
end
|
||||||
9
app/controllers/search_controller.rb
Normal file
9
app/controllers/search_controller.rb
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
class SearchController < ApplicationController
|
||||||
|
def search
|
||||||
|
if params[:term].nil?
|
||||||
|
@pastes = []
|
||||||
|
else
|
||||||
|
@pastes = Paste.search params[:term]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
2
app/helpers/application_helper.rb
Normal file
2
app/helpers/application_helper.rb
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
module ApplicationHelper
|
||||||
|
end
|
||||||
2
app/helpers/pastes_helper.rb
Normal file
2
app/helpers/pastes_helper.rb
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
module PastesHelper
|
||||||
|
end
|
||||||
2
app/helpers/search_helper.rb
Normal file
2
app/helpers/search_helper.rb
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
module SearchHelper
|
||||||
|
end
|
||||||
6
app/javascript/channels/consumer.js
Normal file
6
app/javascript/channels/consumer.js
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
// Action Cable provides the framework to deal with WebSockets in Rails.
|
||||||
|
// You can generate new channels where WebSocket features live using the `rails generate channel` command.
|
||||||
|
|
||||||
|
import { createConsumer } from "@rails/actioncable"
|
||||||
|
|
||||||
|
export default createConsumer()
|
||||||
5
app/javascript/channels/index.js
Normal file
5
app/javascript/channels/index.js
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
// Load all the channels within this directory and all subdirectories.
|
||||||
|
// Channel files must be named *_channel.js.
|
||||||
|
|
||||||
|
const channels = require.context('.', true, /_channel\.js$/)
|
||||||
|
channels.keys().forEach(channels)
|
||||||
17
app/javascript/packs/application.js
Normal file
17
app/javascript/packs/application.js
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
// This file is automatically compiled by Webpack, along with any other files
|
||||||
|
// present in this directory. You're encouraged to place your actual application logic in
|
||||||
|
// a relevant structure within app/javascript and only use these pack files to reference
|
||||||
|
// that code so it'll be compiled.
|
||||||
|
|
||||||
|
require("@rails/ujs").start()
|
||||||
|
require("turbolinks").start()
|
||||||
|
require("@rails/activestorage").start()
|
||||||
|
require("channels")
|
||||||
|
|
||||||
|
|
||||||
|
// Uncomment to copy all static images under ../images to the output folder and reference
|
||||||
|
// them with the image_pack_tag helper in views (e.g <%= image_pack_tag 'rails.png' %>)
|
||||||
|
// or the `imagePath` JavaScript helper below.
|
||||||
|
//
|
||||||
|
// const images = require.context('../images', true)
|
||||||
|
// const imagePath = (name) => images(name, true)
|
||||||
7
app/jobs/application_job.rb
Normal file
7
app/jobs/application_job.rb
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
class ApplicationJob < ActiveJob::Base
|
||||||
|
# Automatically retry jobs that encountered a deadlock
|
||||||
|
# retry_on ActiveRecord::Deadlocked
|
||||||
|
|
||||||
|
# Most jobs are safe to ignore if the underlying records are no longer available
|
||||||
|
# discard_on ActiveJob::DeserializationError
|
||||||
|
end
|
||||||
4
app/mailers/application_mailer.rb
Normal file
4
app/mailers/application_mailer.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
class ApplicationMailer < ActionMailer::Base
|
||||||
|
default from: 'from@example.com'
|
||||||
|
layout 'mailer'
|
||||||
|
end
|
||||||
3
app/models/application_record.rb
Normal file
3
app/models/application_record.rb
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
class ApplicationRecord < ActiveRecord::Base
|
||||||
|
self.abstract_class = true
|
||||||
|
end
|
||||||
0
app/models/concerns/.keep
Normal file
0
app/models/concerns/.keep
Normal file
31
app/models/paste.rb
Normal file
31
app/models/paste.rb
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
require 'elasticsearch/model'
|
||||||
|
|
||||||
|
class Paste < ApplicationRecord
|
||||||
|
include Elasticsearch::Model
|
||||||
|
include Elasticsearch::Model::Callbacks
|
||||||
|
|
||||||
|
|
||||||
|
# this doesnt work yet, but this is how we build a way to progrmatically query ES
|
||||||
|
# def self.search(query)
|
||||||
|
# __elasticsearch__.search(
|
||||||
|
# {
|
||||||
|
# query: {
|
||||||
|
# multi_match: {
|
||||||
|
# query: query,
|
||||||
|
# }
|
||||||
|
# },
|
||||||
|
# "highlight": {
|
||||||
|
# "pre_tags": [
|
||||||
|
# "@kibana-highlighted-field@"
|
||||||
|
# ],
|
||||||
|
# "post_tags": [
|
||||||
|
# "@/kibana-highlighted-field@"
|
||||||
|
# ],
|
||||||
|
# "fields": {
|
||||||
|
# "*": {}
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# )
|
||||||
|
# end
|
||||||
|
end
|
||||||
16
app/views/layouts/application.html.erb
Normal file
16
app/views/layouts/application.html.erb
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Pastebinner</title>
|
||||||
|
<%= csrf_meta_tags %>
|
||||||
|
<%= csp_meta_tag %>
|
||||||
|
|
||||||
|
<%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %>
|
||||||
|
<%= javascript_pack_tag 'application', 'data-turbolinks-track': 'reload' %>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<%= render 'search/form' %>
|
||||||
|
<%= yield %>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
13
app/views/layouts/mailer.html.erb
Normal file
13
app/views/layouts/mailer.html.erb
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||||
|
<style>
|
||||||
|
/* Email styles need to be inline */
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<%= yield %>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
1
app/views/layouts/mailer.text.erb
Normal file
1
app/views/layouts/mailer.text.erb
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<%= yield %>
|
||||||
22
app/views/pastes/_form.html.erb
Normal file
22
app/views/pastes/_form.html.erb
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
<%= form_with(model: paste, local: true) do |form| %>
|
||||||
|
<% if paste.errors.any? %>
|
||||||
|
<div id="error_explanation">
|
||||||
|
<h2><%= pluralize(paste.errors.count, "error") %> prohibited this paste from being saved:</h2>
|
||||||
|
|
||||||
|
<ul>
|
||||||
|
<% paste.errors.full_messages.each do |message| %>
|
||||||
|
<li><%= message %></li>
|
||||||
|
<% end %>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
|
<div class="field">
|
||||||
|
<%= form.label :search_pastes %>
|
||||||
|
<%= form.text_area :search_pastes %>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="actions">
|
||||||
|
<%= form.submit %>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
2
app/views/pastes/_paste.json.jbuilder
Normal file
2
app/views/pastes/_paste.json.jbuilder
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
json.extract! paste, :id, :search_pastes, :created_at, :updated_at
|
||||||
|
json.url paste_url(paste, format: :json)
|
||||||
6
app/views/pastes/edit.html.erb
Normal file
6
app/views/pastes/edit.html.erb
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
<h1>Editing Paste</h1>
|
||||||
|
|
||||||
|
<%= render 'form', paste: @paste %>
|
||||||
|
|
||||||
|
<%= link_to 'Show', @paste %> |
|
||||||
|
<%= link_to 'Back', pastes_path %>
|
||||||
27
app/views/pastes/index.html.erb
Normal file
27
app/views/pastes/index.html.erb
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
<p id="notice"><%= notice %></p>
|
||||||
|
|
||||||
|
<h1>Pastes</h1>
|
||||||
|
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Search pastes</th>
|
||||||
|
<th colspan="3"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
|
||||||
|
<tbody>
|
||||||
|
<% @pastes.each do |paste| %>
|
||||||
|
<tr>
|
||||||
|
<td><%= paste.search_pastes %></td>
|
||||||
|
<td><%= link_to 'Show', paste %></td>
|
||||||
|
<td><%= link_to 'Edit', edit_paste_path(paste) %></td>
|
||||||
|
<td><%= link_to 'Destroy', paste, method: :delete, data: { confirm: 'Are you sure?' } %></td>
|
||||||
|
</tr>
|
||||||
|
<% end %>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<br>
|
||||||
|
|
||||||
|
<%= link_to 'New Paste', new_paste_path %>
|
||||||
1
app/views/pastes/index.json.jbuilder
Normal file
1
app/views/pastes/index.json.jbuilder
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
json.array! @pastes, partial: "pastes/paste", as: :paste
|
||||||
5
app/views/pastes/new.html.erb
Normal file
5
app/views/pastes/new.html.erb
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
<h1>New Paste</h1>
|
||||||
|
|
||||||
|
<%= render 'form', paste: @paste %>
|
||||||
|
|
||||||
|
<%= link_to 'Back', pastes_path %>
|
||||||
9
app/views/pastes/show.html.erb
Normal file
9
app/views/pastes/show.html.erb
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
<p id="notice"><%= notice %></p>
|
||||||
|
|
||||||
|
<p>
|
||||||
|
<strong>Search pastes:</strong>
|
||||||
|
<%= @paste.search_pastes %>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<%= link_to 'Edit', edit_paste_path(@paste) %> |
|
||||||
|
<%= link_to 'Back', pastes_path %>
|
||||||
1
app/views/pastes/show.json.jbuilder
Normal file
1
app/views/pastes/show.json.jbuilder
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
json.partial! "pastes/paste", paste: @paste
|
||||||
29
app/views/search/_form.html.erb
Normal file
29
app/views/search/_form.html.erb
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
<%= form_for :term, url: search_path, method: :get do |form| %>
|
||||||
|
<p>
|
||||||
|
<%= text_field_tag :term, params[:term] %>
|
||||||
|
<%= submit_tag "Search", name: nil %>
|
||||||
|
</p>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
|
<h1>Search Results</h1>
|
||||||
|
|
||||||
|
<% if @pastes %>
|
||||||
|
<ul class="search_results">
|
||||||
|
<% @pastes.each do |paste| %>
|
||||||
|
<li>
|
||||||
|
<h3>
|
||||||
|
<%= link_to article.try(:highlight).try(:title) ?
|
||||||
|
paste.highlight.title[0].html_safe : paste.title,
|
||||||
|
controller: "pastes", action: "show", id: paste._id %>
|
||||||
|
</h3>
|
||||||
|
<% if paste.try(:highlight).try(:text) %>
|
||||||
|
<% paste.highlight.text.each do |snippet| %>
|
||||||
|
<p><%= snippet.html_safe %>...</p>
|
||||||
|
<% end %>
|
||||||
|
<% end %>
|
||||||
|
</li>
|
||||||
|
<% end %>
|
||||||
|
</ul>
|
||||||
|
<% else %>
|
||||||
|
<p>Your search did not match any documents.</p>
|
||||||
|
<% end %>
|
||||||
10
app/workers/send_pastes_to_elastic_search_worker.rb
Normal file
10
app/workers/send_pastes_to_elastic_search_worker.rb
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
class SendPastesToElasticSearchWorker
|
||||||
|
include Sidekiq::Worker
|
||||||
|
sidekiq_options retry: false
|
||||||
|
|
||||||
|
def perform
|
||||||
|
es_helper = EsSearchHelper.new('http://pastebinner-elasticsearch:9200', 'pastes')
|
||||||
|
# we are sending 2 requests per arg here so 100 to sends_json_to_es is 200 requests
|
||||||
|
es_helper.send_jsons_to_es(50)
|
||||||
|
end
|
||||||
|
end
|
||||||
72
babel.config.js
Normal file
72
babel.config.js
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
module.exports = function(api) {
|
||||||
|
var validEnv = ['development', 'test', 'production']
|
||||||
|
var currentEnv = api.env()
|
||||||
|
var isDevelopmentEnv = api.env('development')
|
||||||
|
var isProductionEnv = api.env('production')
|
||||||
|
var isTestEnv = api.env('test')
|
||||||
|
|
||||||
|
if (!validEnv.includes(currentEnv)) {
|
||||||
|
throw new Error(
|
||||||
|
'Please specify a valid `NODE_ENV` or ' +
|
||||||
|
'`BABEL_ENV` environment variables. Valid values are "development", ' +
|
||||||
|
'"test", and "production". Instead, received: ' +
|
||||||
|
JSON.stringify(currentEnv) +
|
||||||
|
'.'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
presets: [
|
||||||
|
isTestEnv && [
|
||||||
|
require('@babel/preset-env').default,
|
||||||
|
{
|
||||||
|
targets: {
|
||||||
|
node: 'current'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
(isProductionEnv || isDevelopmentEnv) && [
|
||||||
|
require('@babel/preset-env').default,
|
||||||
|
{
|
||||||
|
forceAllTransforms: true,
|
||||||
|
useBuiltIns: 'entry',
|
||||||
|
corejs: 3,
|
||||||
|
modules: false,
|
||||||
|
exclude: ['transform-typeof-symbol']
|
||||||
|
}
|
||||||
|
]
|
||||||
|
].filter(Boolean),
|
||||||
|
plugins: [
|
||||||
|
require('babel-plugin-macros'),
|
||||||
|
require('@babel/plugin-syntax-dynamic-import').default,
|
||||||
|
isTestEnv && require('babel-plugin-dynamic-import-node'),
|
||||||
|
require('@babel/plugin-transform-destructuring').default,
|
||||||
|
[
|
||||||
|
require('@babel/plugin-proposal-class-properties').default,
|
||||||
|
{
|
||||||
|
loose: true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[
|
||||||
|
require('@babel/plugin-proposal-object-rest-spread').default,
|
||||||
|
{
|
||||||
|
useBuiltIns: true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[
|
||||||
|
require('@babel/plugin-transform-runtime').default,
|
||||||
|
{
|
||||||
|
helpers: false,
|
||||||
|
regenerator: true,
|
||||||
|
corejs: false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[
|
||||||
|
require('@babel/plugin-transform-regenerator').default,
|
||||||
|
{
|
||||||
|
async: false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
].filter(Boolean)
|
||||||
|
}
|
||||||
|
}
|
||||||
105
bin/bundle
Executable file
105
bin/bundle
Executable file
|
|
@ -0,0 +1,105 @@
|
||||||
|
#!/usr/bin/env ruby2.5
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
#
|
||||||
|
# This file was generated by Bundler.
|
||||||
|
#
|
||||||
|
# The application 'bundle' is installed as part of a gem, and
|
||||||
|
# this file is here to facilitate running it.
|
||||||
|
#
|
||||||
|
|
||||||
|
require "rubygems"
|
||||||
|
|
||||||
|
m = Module.new do
|
||||||
|
module_function
|
||||||
|
|
||||||
|
def invoked_as_script?
|
||||||
|
File.expand_path($0) == File.expand_path(__FILE__)
|
||||||
|
end
|
||||||
|
|
||||||
|
def env_var_version
|
||||||
|
ENV["BUNDLER_VERSION"]
|
||||||
|
end
|
||||||
|
|
||||||
|
def cli_arg_version
|
||||||
|
return unless invoked_as_script? # don't want to hijack other binstubs
|
||||||
|
return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update`
|
||||||
|
bundler_version = nil
|
||||||
|
update_index = nil
|
||||||
|
ARGV.each_with_index do |a, i|
|
||||||
|
if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN
|
||||||
|
bundler_version = a
|
||||||
|
end
|
||||||
|
next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/
|
||||||
|
bundler_version = $1 || ">= 0.a"
|
||||||
|
update_index = i
|
||||||
|
end
|
||||||
|
bundler_version
|
||||||
|
end
|
||||||
|
|
||||||
|
def gemfile
|
||||||
|
gemfile = ENV["BUNDLE_GEMFILE"]
|
||||||
|
return gemfile if gemfile && !gemfile.empty?
|
||||||
|
|
||||||
|
File.expand_path("../../Gemfile", __FILE__)
|
||||||
|
end
|
||||||
|
|
||||||
|
def lockfile
|
||||||
|
lockfile =
|
||||||
|
case File.basename(gemfile)
|
||||||
|
when "gems.rb" then gemfile.sub(/\.rb$/, gemfile)
|
||||||
|
else "#{gemfile}.lock"
|
||||||
|
end
|
||||||
|
File.expand_path(lockfile)
|
||||||
|
end
|
||||||
|
|
||||||
|
def lockfile_version
|
||||||
|
return unless File.file?(lockfile)
|
||||||
|
lockfile_contents = File.read(lockfile)
|
||||||
|
return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/
|
||||||
|
Regexp.last_match(1)
|
||||||
|
end
|
||||||
|
|
||||||
|
def bundler_version
|
||||||
|
@bundler_version ||= begin
|
||||||
|
env_var_version || cli_arg_version ||
|
||||||
|
lockfile_version || "#{Gem::Requirement.default}.a"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def load_bundler!
|
||||||
|
ENV["BUNDLE_GEMFILE"] ||= gemfile
|
||||||
|
|
||||||
|
# must dup string for RG < 1.8 compatibility
|
||||||
|
activate_bundler(bundler_version.dup)
|
||||||
|
end
|
||||||
|
|
||||||
|
def activate_bundler(bundler_version)
|
||||||
|
if Gem::Version.correct?(bundler_version) && Gem::Version.new(bundler_version).release < Gem::Version.new("2.0")
|
||||||
|
bundler_version = "< 2"
|
||||||
|
end
|
||||||
|
gem_error = activation_error_handling do
|
||||||
|
gem "bundler", bundler_version
|
||||||
|
end
|
||||||
|
return if gem_error.nil?
|
||||||
|
require_error = activation_error_handling do
|
||||||
|
require "bundler/version"
|
||||||
|
end
|
||||||
|
return if require_error.nil? && Gem::Requirement.new(bundler_version).satisfied_by?(Gem::Version.new(Bundler::VERSION))
|
||||||
|
warn "Activating bundler (#{bundler_version}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_version}'`"
|
||||||
|
exit 42
|
||||||
|
end
|
||||||
|
|
||||||
|
def activation_error_handling
|
||||||
|
yield
|
||||||
|
nil
|
||||||
|
rescue StandardError, LoadError => e
|
||||||
|
e
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
m.load_bundler!
|
||||||
|
|
||||||
|
if m.invoked_as_script?
|
||||||
|
load Gem.bin_path("bundler", "bundle")
|
||||||
|
end
|
||||||
9
bin/rails
Executable file
9
bin/rails
Executable file
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
begin
|
||||||
|
load File.expand_path('../spring', __FILE__)
|
||||||
|
rescue LoadError => e
|
||||||
|
raise unless e.message.include?('spring')
|
||||||
|
end
|
||||||
|
APP_PATH = File.expand_path('../config/application', __dir__)
|
||||||
|
require_relative '../config/boot'
|
||||||
|
require 'rails/commands'
|
||||||
9
bin/rake
Executable file
9
bin/rake
Executable file
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
begin
|
||||||
|
load File.expand_path('../spring', __FILE__)
|
||||||
|
rescue LoadError => e
|
||||||
|
raise unless e.message.include?('spring')
|
||||||
|
end
|
||||||
|
require_relative '../config/boot'
|
||||||
|
require 'rake'
|
||||||
|
Rake.application.run
|
||||||
40
bin/setup
40
bin/setup
|
|
@ -1,8 +1,36 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env ruby
|
||||||
set -euo pipefail
|
require 'fileutils'
|
||||||
IFS=$'\n\t'
|
|
||||||
set -vx
|
|
||||||
|
|
||||||
bundle install
|
# path to your application root.
|
||||||
|
APP_ROOT = File.expand_path('..', __dir__)
|
||||||
|
|
||||||
# Do any other automated setup that you need to do here
|
def system!(*args)
|
||||||
|
system(*args) || abort("\n== Command #{args} failed ==")
|
||||||
|
end
|
||||||
|
|
||||||
|
FileUtils.chdir APP_ROOT do
|
||||||
|
# This script is a way to setup or update your development environment automatically.
|
||||||
|
# This script is idempotent, so that you can run it at anytime and get an expectable outcome.
|
||||||
|
# Add necessary setup steps to this file.
|
||||||
|
|
||||||
|
puts '== Installing dependencies =='
|
||||||
|
system! 'gem install bundler --conservative'
|
||||||
|
system('bundle check') || system!('bundle install')
|
||||||
|
|
||||||
|
# Install JavaScript dependencies
|
||||||
|
# system('bin/yarn')
|
||||||
|
|
||||||
|
# puts "\n== Copying sample files =="
|
||||||
|
# unless File.exist?('config/database.yml')
|
||||||
|
# FileUtils.cp 'config/database.yml.sample', 'config/database.yml'
|
||||||
|
# end
|
||||||
|
|
||||||
|
puts "\n== Preparing database =="
|
||||||
|
system! 'bin/rails db:prepare'
|
||||||
|
|
||||||
|
puts "\n== Removing old logs and tempfiles =="
|
||||||
|
system! 'bin/rails log:clear tmp:clear'
|
||||||
|
|
||||||
|
puts "\n== Restarting application server =="
|
||||||
|
system! 'bin/rails restart'
|
||||||
|
end
|
||||||
|
|
|
||||||
17
bin/spring
Executable file
17
bin/spring
Executable file
|
|
@ -0,0 +1,17 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
# This file loads Spring without using Bundler, in order to be fast.
|
||||||
|
# It gets overwritten when you run the `spring binstub` command.
|
||||||
|
|
||||||
|
unless defined?(Spring)
|
||||||
|
require 'rubygems'
|
||||||
|
require 'bundler'
|
||||||
|
|
||||||
|
lockfile = Bundler::LockfileParser.new(Bundler.default_lockfile.read)
|
||||||
|
spring = lockfile.specs.detect { |spec| spec.name == 'spring' }
|
||||||
|
if spring
|
||||||
|
Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path
|
||||||
|
gem 'spring', spring.version
|
||||||
|
require 'spring/binstub'
|
||||||
|
end
|
||||||
|
end
|
||||||
19
bin/webpack
Executable file
19
bin/webpack
Executable file
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
ENV["RAILS_ENV"] ||= ENV["RACK_ENV"] || "development"
|
||||||
|
ENV["NODE_ENV"] ||= "development"
|
||||||
|
|
||||||
|
require "pathname"
|
||||||
|
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
|
||||||
|
Pathname.new(__FILE__).realpath)
|
||||||
|
|
||||||
|
require "rubygems"
|
||||||
|
require "bundler/setup"
|
||||||
|
|
||||||
|
require "webpacker"
|
||||||
|
require "webpacker/webpack_runner"
|
||||||
|
|
||||||
|
APP_ROOT = File.expand_path("..", __dir__)
|
||||||
|
Dir.chdir(APP_ROOT) do
|
||||||
|
Webpacker::WebpackRunner.run(ARGV)
|
||||||
|
end
|
||||||
19
bin/webpack-dev-server
Executable file
19
bin/webpack-dev-server
Executable file
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
ENV["RAILS_ENV"] ||= ENV["RACK_ENV"] || "development"
|
||||||
|
ENV["NODE_ENV"] ||= "development"
|
||||||
|
|
||||||
|
require "pathname"
|
||||||
|
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
|
||||||
|
Pathname.new(__FILE__).realpath)
|
||||||
|
|
||||||
|
require "rubygems"
|
||||||
|
require "bundler/setup"
|
||||||
|
|
||||||
|
require "webpacker"
|
||||||
|
require "webpacker/dev_server_runner"
|
||||||
|
|
||||||
|
APP_ROOT = File.expand_path("..", __dir__)
|
||||||
|
Dir.chdir(APP_ROOT) do
|
||||||
|
Webpacker::DevServerRunner.run(ARGV)
|
||||||
|
end
|
||||||
11
bin/yarn
Executable file
11
bin/yarn
Executable file
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
APP_ROOT = File.expand_path('..', __dir__)
|
||||||
|
Dir.chdir(APP_ROOT) do
|
||||||
|
begin
|
||||||
|
exec "yarnpkg", *ARGV
|
||||||
|
rescue Errno::ENOENT
|
||||||
|
$stderr.puts "Yarn executable was not detected in the system."
|
||||||
|
$stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install"
|
||||||
|
exit 1
|
||||||
|
end
|
||||||
|
end
|
||||||
10
config.ru
Normal file
10
config.ru
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# This file is used by Rack-based servers to start the application.
|
||||||
|
|
||||||
|
require_relative 'config/environment'
|
||||||
|
|
||||||
|
require 'sidekiq/web'
|
||||||
|
require 'sidekiq/cron/web'
|
||||||
|
|
||||||
|
run Sidekiq::Web
|
||||||
|
|
||||||
|
run Rails.application
|
||||||
27
config/application.rb
Normal file
27
config/application.rb
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
require_relative 'boot'
|
||||||
|
|
||||||
|
require 'rails/all'
|
||||||
|
|
||||||
|
# Require the gems listed in Gemfile, including any gems
|
||||||
|
# you've limited to :test, :development, or :production.
|
||||||
|
Bundler.require(*Rails.groups)
|
||||||
|
|
||||||
|
module Pastebinner
|
||||||
|
class Application < Rails::Application
|
||||||
|
# Initialize configuration defaults for originally generated Rails version.
|
||||||
|
config.load_defaults 6.0
|
||||||
|
# Settings in config/environments/* take precedence over those specified here.
|
||||||
|
# Application configuration can go into files in config/initializers
|
||||||
|
config.paths.add Rails.root.join('lib').to_s, eager_load: true
|
||||||
|
config.active_job.queue_adapter = :sidekiq
|
||||||
|
# -- all .rb files in that directory are automatically loaded after loading
|
||||||
|
# the framework and any gems in your application.
|
||||||
|
# uncomment the config block below to have environment vars set from a local_env.yml file.
|
||||||
|
# config.before_configuration do
|
||||||
|
# env_file = File.join(Rails.root, 'config', 'local_env.yml')
|
||||||
|
# YAML.load(File.open(env_file)).each do |key, value|
|
||||||
|
# ENV[key.to_s] = value
|
||||||
|
# end if File.exists?(env_file)
|
||||||
|
# end
|
||||||
|
end
|
||||||
|
end
|
||||||
4
config/boot.rb
Normal file
4
config/boot.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
|
||||||
|
|
||||||
|
require 'bundler/setup' # Set up gems listed in the Gemfile.
|
||||||
|
require 'bootsnap/setup' # Speed up boot time by caching expensive operations.
|
||||||
10
config/cable.yml
Normal file
10
config/cable.yml
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
development:
|
||||||
|
adapter: async
|
||||||
|
|
||||||
|
test:
|
||||||
|
adapter: test
|
||||||
|
|
||||||
|
production:
|
||||||
|
adapter: redis
|
||||||
|
url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
|
||||||
|
channel_prefix: pastebinner_production
|
||||||
1
config/credentials.yml.enc
Normal file
1
config/credentials.yml.enc
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
welMzjpF+fFBB25lKGiSxBqcltfMDS9mr7e8Vnhso/EcpdiBtHbhnm/nMboyEdvwcjeHUgSvmzIOiiQNcWigQsz12C5+quHmmvxhzu+5RCwQ37KB5vGZcNkaBYMt2ea5HhRZV8ssyn4zD+pmlQDy9EFvjvph6YgyCxYGbDqFVA4D2mQk7hXhd8rVEQ6J12uTyj3YjzoNf3SbiskM6vh8FT03mChmAmVxy/lMmSGdmwItxPKP6uWwiLhpHBosFW6wmPY/QqRnwgVEonVA80SD1K5fvaq3WElS1KvfawNNV35DdC2kSpW0N/KDRuxnb/QVbPBfvLFxDJcOpP84wdtSHYVqZqXt7lnN7WtVZ6vYd0DoJXVODYi2cPR4MEPZf9d1CmSz0nIyy3kEdHjT6eLra2yXhaKnaKsO+AhZ--PqwJ2t+O+9PPK6M5--6CRyG5E2k6TLqB/AJ2bWZg==
|
||||||
25
config/database.yml
Normal file
25
config/database.yml
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
# SQLite. Versions 3.8.0 and up are supported.
|
||||||
|
# gem install sqlite3
|
||||||
|
#
|
||||||
|
# Ensure the SQLite 3 gem is defined in your Gemfile
|
||||||
|
# gem 'sqlite3'
|
||||||
|
#
|
||||||
|
default: &default
|
||||||
|
adapter: sqlite3
|
||||||
|
pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
|
||||||
|
timeout: 5000
|
||||||
|
|
||||||
|
development:
|
||||||
|
<<: *default
|
||||||
|
database: db/development.sqlite3
|
||||||
|
|
||||||
|
# Warning: The database defined as "test" will be erased and
|
||||||
|
# re-generated from your development database when you run "rake".
|
||||||
|
# Do not set this db to the same as development or production.
|
||||||
|
test:
|
||||||
|
<<: *default
|
||||||
|
database: db/test.sqlite3
|
||||||
|
|
||||||
|
production:
|
||||||
|
<<: *default
|
||||||
|
database: db/production.sqlite3
|
||||||
5
config/environment.rb
Normal file
5
config/environment.rb
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
# Load the Rails application.
|
||||||
|
require_relative 'application'
|
||||||
|
|
||||||
|
# Initialize the Rails application.
|
||||||
|
Rails.application.initialize!
|
||||||
65
config/environments/development.rb
Normal file
65
config/environments/development.rb
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
Rails.application.configure do
|
||||||
|
# Settings specified here will take precedence over those in config/application.rb.
|
||||||
|
|
||||||
|
# In the development environment your application's code is reloaded on
|
||||||
|
# every request. This slows down response time but is perfect for development
|
||||||
|
# since you don't have to restart the web server when you make code changes.
|
||||||
|
|
||||||
|
config.console = Pry
|
||||||
|
config.hosts << "pastebinner.mcdevitt.tech"
|
||||||
|
config.cache_classes = false
|
||||||
|
|
||||||
|
# Do not eager load code on boot.
|
||||||
|
config.eager_load = false
|
||||||
|
|
||||||
|
# Show full error reports.
|
||||||
|
config.consider_all_requests_local = true
|
||||||
|
|
||||||
|
# Enable/disable caching. By default caching is disabled.
|
||||||
|
# Run rails dev:cache to toggle caching.
|
||||||
|
if Rails.root.join('tmp', 'caching-dev.txt').exist?
|
||||||
|
config.action_controller.perform_caching = true
|
||||||
|
config.action_controller.enable_fragment_cache_logging = true
|
||||||
|
|
||||||
|
config.cache_store = :memory_store
|
||||||
|
config.public_file_server.headers = {
|
||||||
|
'Cache-Control' => "public, max-age=#{2.days.to_i}"
|
||||||
|
}
|
||||||
|
else
|
||||||
|
config.action_controller.perform_caching = false
|
||||||
|
|
||||||
|
config.cache_store = :null_store
|
||||||
|
end
|
||||||
|
|
||||||
|
# Store uploaded files on the local file system (see config/storage.yml for options).
|
||||||
|
config.active_storage.service = :local
|
||||||
|
|
||||||
|
# Don't care if the mailer can't send.
|
||||||
|
config.action_mailer.raise_delivery_errors = false
|
||||||
|
|
||||||
|
config.action_mailer.perform_caching = false
|
||||||
|
|
||||||
|
# Print deprecation notices to the Rails logger.
|
||||||
|
config.active_support.deprecation = :log
|
||||||
|
|
||||||
|
# Raise an error on page load if there are pending migrations.
|
||||||
|
config.active_record.migration_error = :page_load
|
||||||
|
|
||||||
|
# Highlight code that triggered database queries in logs.
|
||||||
|
config.active_record.verbose_query_logs = true
|
||||||
|
|
||||||
|
# Debug mode disables concatenation and preprocessing of assets.
|
||||||
|
# This option may cause significant delays in view rendering with a large
|
||||||
|
# number of complex assets.
|
||||||
|
config.assets.debug = true
|
||||||
|
|
||||||
|
# Suppress logger output for asset requests.
|
||||||
|
config.assets.quiet = true
|
||||||
|
|
||||||
|
# Raises error for missing translations.
|
||||||
|
# config.action_view.raise_on_missing_translations = true
|
||||||
|
|
||||||
|
# Use an evented file watcher to asynchronously detect changes in source code,
|
||||||
|
# routes, locales, etc. This feature depends on the listen gem.
|
||||||
|
config.file_watcher = ActiveSupport::EventedFileUpdateChecker
|
||||||
|
end
|
||||||
112
config/environments/production.rb
Normal file
112
config/environments/production.rb
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
Rails.application.configure do
|
||||||
|
# Settings specified here will take precedence over those in config/application.rb.
|
||||||
|
|
||||||
|
# Code is not reloaded between requests.
|
||||||
|
config.cache_classes = true
|
||||||
|
|
||||||
|
# Eager load code on boot. This eager loads most of Rails and
|
||||||
|
# your application in memory, allowing both threaded web servers
|
||||||
|
# and those relying on copy on write to perform better.
|
||||||
|
# Rake tasks automatically ignore this option for performance.
|
||||||
|
config.eager_load = true
|
||||||
|
|
||||||
|
# Full error reports are disabled and caching is turned on.
|
||||||
|
config.consider_all_requests_local = false
|
||||||
|
config.action_controller.perform_caching = true
|
||||||
|
|
||||||
|
# Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
|
||||||
|
# or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
|
||||||
|
# config.require_master_key = true
|
||||||
|
|
||||||
|
# Disable serving static files from the `/public` folder by default since
|
||||||
|
# Apache or NGINX already handles this.
|
||||||
|
config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
|
||||||
|
|
||||||
|
# Compress CSS using a preprocessor.
|
||||||
|
# config.assets.css_compressor = :sass
|
||||||
|
|
||||||
|
# Do not fallback to assets pipeline if a precompiled asset is missed.
|
||||||
|
config.assets.compile = false
|
||||||
|
|
||||||
|
# Enable serving of images, stylesheets, and JavaScripts from an asset server.
|
||||||
|
# config.action_controller.asset_host = 'http://assets.example.com'
|
||||||
|
|
||||||
|
# Specifies the header that your server uses for sending files.
|
||||||
|
# config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
|
||||||
|
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
|
||||||
|
|
||||||
|
# Store uploaded files on the local file system (see config/storage.yml for options).
|
||||||
|
config.active_storage.service = :local
|
||||||
|
|
||||||
|
# Mount Action Cable outside main process or domain.
|
||||||
|
# config.action_cable.mount_path = nil
|
||||||
|
# config.action_cable.url = 'wss://example.com/cable'
|
||||||
|
# config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ]
|
||||||
|
|
||||||
|
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
|
||||||
|
# config.force_ssl = true
|
||||||
|
|
||||||
|
# Use the lowest log level to ensure availability of diagnostic information
|
||||||
|
# when problems arise.
|
||||||
|
config.log_level = :debug
|
||||||
|
|
||||||
|
# Prepend all log lines with the following tags.
|
||||||
|
config.log_tags = [ :request_id ]
|
||||||
|
|
||||||
|
# Use a different cache store in production.
|
||||||
|
# config.cache_store = :mem_cache_store
|
||||||
|
|
||||||
|
# Use a real queuing backend for Active Job (and separate queues per environment).
|
||||||
|
# config.active_job.queue_adapter = :resque
|
||||||
|
# config.active_job.queue_name_prefix = "pastebinner_production"
|
||||||
|
|
||||||
|
config.action_mailer.perform_caching = false
|
||||||
|
|
||||||
|
# Ignore bad email addresses and do not raise email delivery errors.
|
||||||
|
# Set this to true and configure the email server for immediate delivery to raise delivery errors.
|
||||||
|
# config.action_mailer.raise_delivery_errors = false
|
||||||
|
|
||||||
|
# Enable locale fallbacks for I18n (makes lookups for any locale fall back to
|
||||||
|
# the I18n.default_locale when a translation cannot be found).
|
||||||
|
config.i18n.fallbacks = true
|
||||||
|
|
||||||
|
# Send deprecation notices to registered listeners.
|
||||||
|
config.active_support.deprecation = :notify
|
||||||
|
|
||||||
|
# Use default logging formatter so that PID and timestamp are not suppressed.
|
||||||
|
config.log_formatter = ::Logger::Formatter.new
|
||||||
|
|
||||||
|
# Use a different logger for distributed setups.
|
||||||
|
# require 'syslog/logger'
|
||||||
|
# config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
|
||||||
|
|
||||||
|
if ENV["RAILS_LOG_TO_STDOUT"].present?
|
||||||
|
logger = ActiveSupport::Logger.new(STDOUT)
|
||||||
|
logger.formatter = config.log_formatter
|
||||||
|
config.logger = ActiveSupport::TaggedLogging.new(logger)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Do not dump schema after migrations.
|
||||||
|
config.active_record.dump_schema_after_migration = false
|
||||||
|
|
||||||
|
# Inserts middleware to perform automatic connection switching.
|
||||||
|
# The `database_selector` hash is used to pass options to the DatabaseSelector
|
||||||
|
# middleware. The `delay` is used to determine how long to wait after a write
|
||||||
|
# to send a subsequent read to the primary.
|
||||||
|
#
|
||||||
|
# The `database_resolver` class is used by the middleware to determine which
|
||||||
|
# database is appropriate to use based on the time delay.
|
||||||
|
#
|
||||||
|
# The `database_resolver_context` class is used by the middleware to set
|
||||||
|
# timestamps for the last write to the primary. The resolver uses the context
|
||||||
|
# class timestamps to determine how long to wait before reading from the
|
||||||
|
# replica.
|
||||||
|
#
|
||||||
|
# By default Rails will store a last write timestamp in the session. The
|
||||||
|
# DatabaseSelector middleware is designed as such you can define your own
|
||||||
|
# strategy for connection switching and pass that into the middleware through
|
||||||
|
# these configuration options.
|
||||||
|
# config.active_record.database_selector = { delay: 2.seconds }
|
||||||
|
# config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver
|
||||||
|
# config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session
|
||||||
|
end
|
||||||
48
config/environments/test.rb
Normal file
48
config/environments/test.rb
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
# The test environment is used exclusively to run your application's
|
||||||
|
# test suite. You never need to work with it otherwise. Remember that
|
||||||
|
# your test database is "scratch space" for the test suite and is wiped
|
||||||
|
# and recreated between test runs. Don't rely on the data there!
|
||||||
|
|
||||||
|
Rails.application.configure do
|
||||||
|
# Settings specified here will take precedence over those in config/application.rb.
|
||||||
|
|
||||||
|
config.cache_classes = false
|
||||||
|
|
||||||
|
# Do not eager load code on boot. This avoids loading your whole application
|
||||||
|
# just for the purpose of running a single test. If you are using a tool that
|
||||||
|
# preloads Rails for running tests, you may have to set it to true.
|
||||||
|
config.eager_load = false
|
||||||
|
|
||||||
|
# Configure public file server for tests with Cache-Control for performance.
|
||||||
|
config.public_file_server.enabled = true
|
||||||
|
config.public_file_server.headers = {
|
||||||
|
'Cache-Control' => "public, max-age=#{1.hour.to_i}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Show full error reports and disable caching.
|
||||||
|
config.consider_all_requests_local = true
|
||||||
|
config.action_controller.perform_caching = false
|
||||||
|
config.cache_store = :null_store
|
||||||
|
|
||||||
|
# Raise exceptions instead of rendering exception templates.
|
||||||
|
config.action_dispatch.show_exceptions = false
|
||||||
|
|
||||||
|
# Disable request forgery protection in test environment.
|
||||||
|
config.action_controller.allow_forgery_protection = false
|
||||||
|
|
||||||
|
# Store uploaded files on the local file system in a temporary directory.
|
||||||
|
config.active_storage.service = :test
|
||||||
|
|
||||||
|
config.action_mailer.perform_caching = false
|
||||||
|
|
||||||
|
# Tell Action Mailer not to deliver emails to the real world.
|
||||||
|
# The :test delivery method accumulates sent emails in the
|
||||||
|
# ActionMailer::Base.deliveries array.
|
||||||
|
config.action_mailer.delivery_method = :test
|
||||||
|
|
||||||
|
# Print deprecation notices to the stderr.
|
||||||
|
config.active_support.deprecation = :stderr
|
||||||
|
|
||||||
|
# Raises error for missing translations.
|
||||||
|
# config.action_view.raise_on_missing_translations = true
|
||||||
|
end
|
||||||
8
config/initializers/application_controller_renderer.rb
Normal file
8
config/initializers/application_controller_renderer.rb
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# ActiveSupport::Reloader.to_prepare do
|
||||||
|
# ApplicationController.renderer.defaults.merge!(
|
||||||
|
# http_host: 'example.org',
|
||||||
|
# https: false
|
||||||
|
# )
|
||||||
|
# end
|
||||||
14
config/initializers/assets.rb
Normal file
14
config/initializers/assets.rb
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Version of your assets, change this if you want to expire all your assets.
|
||||||
|
Rails.application.config.assets.version = '1.0'
|
||||||
|
|
||||||
|
# Add additional assets to the asset load path.
|
||||||
|
# Rails.application.config.assets.paths << Emoji.images_path
|
||||||
|
# Add Yarn node_modules folder to the asset load path.
|
||||||
|
Rails.application.config.assets.paths << Rails.root.join('node_modules')
|
||||||
|
|
||||||
|
# Precompile additional assets.
|
||||||
|
# application.js, application.css, and all non-JS/CSS in the app/assets
|
||||||
|
# folder are already added.
|
||||||
|
# Rails.application.config.assets.precompile += %w( admin.js admin.css )
|
||||||
7
config/initializers/backtrace_silencers.rb
Normal file
7
config/initializers/backtrace_silencers.rb
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
|
||||||
|
# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
|
||||||
|
|
||||||
|
# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
|
||||||
|
# Rails.backtrace_cleaner.remove_silencers!
|
||||||
30
config/initializers/content_security_policy.rb
Normal file
30
config/initializers/content_security_policy.rb
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Define an application-wide content security policy
|
||||||
|
# For further information see the following documentation
|
||||||
|
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
|
||||||
|
|
||||||
|
# Rails.application.config.content_security_policy do |policy|
|
||||||
|
# policy.default_src :self, :https
|
||||||
|
# policy.font_src :self, :https, :data
|
||||||
|
# policy.img_src :self, :https, :data
|
||||||
|
# policy.object_src :none
|
||||||
|
# policy.script_src :self, :https
|
||||||
|
# policy.style_src :self, :https
|
||||||
|
# # If you are using webpack-dev-server then specify webpack-dev-server host
|
||||||
|
# policy.connect_src :self, :https, "http://localhost:3035", "ws://localhost:3035" if Rails.env.development?
|
||||||
|
|
||||||
|
# # Specify URI for violation reports
|
||||||
|
# # policy.report_uri "/csp-violation-report-endpoint"
|
||||||
|
# end
|
||||||
|
|
||||||
|
# If you are using UJS then enable automatic nonce generation
|
||||||
|
# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) }
|
||||||
|
|
||||||
|
# Set the nonce only to specific directives
|
||||||
|
# Rails.application.config.content_security_policy_nonce_directives = %w(script-src)
|
||||||
|
|
||||||
|
# Report CSP violations to a specified URI
|
||||||
|
# For further information see the following documentation:
|
||||||
|
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only
|
||||||
|
# Rails.application.config.content_security_policy_report_only = true
|
||||||
5
config/initializers/cookies_serializer.rb
Normal file
5
config/initializers/cookies_serializer.rb
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Specify a serializer for the signed and encrypted cookie jars.
|
||||||
|
# Valid options are :json, :marshal, and :hybrid.
|
||||||
|
Rails.application.config.action_dispatch.cookies_serializer = :json
|
||||||
4
config/initializers/filter_parameter_logging.rb
Normal file
4
config/initializers/filter_parameter_logging.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Configure sensitive parameters which will be filtered from the log file.
|
||||||
|
Rails.application.config.filter_parameters += [:password]
|
||||||
16
config/initializers/inflections.rb
Normal file
16
config/initializers/inflections.rb
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Add new inflection rules using the following format. Inflections
|
||||||
|
# are locale specific, and you may define rules for as many different
|
||||||
|
# locales as you wish. All of these examples are active by default:
|
||||||
|
# ActiveSupport::Inflector.inflections(:en) do |inflect|
|
||||||
|
# inflect.plural /^(ox)$/i, '\1en'
|
||||||
|
# inflect.singular /^(ox)en/i, '\1'
|
||||||
|
# inflect.irregular 'person', 'people'
|
||||||
|
# inflect.uncountable %w( fish sheep )
|
||||||
|
# end
|
||||||
|
|
||||||
|
# These inflection rules are supported but not enabled by default:
|
||||||
|
# ActiveSupport::Inflector.inflections(:en) do |inflect|
|
||||||
|
# inflect.acronym 'RESTful'
|
||||||
|
# end
|
||||||
4
config/initializers/mime_types.rb
Normal file
4
config/initializers/mime_types.rb
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# Add new mime types for use in respond_to blocks:
|
||||||
|
# Mime::Type.register "text/richtext", :rtf
|
||||||
15
config/initializers/sidekiq.rb
Normal file
15
config/initializers/sidekiq.rb
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
# server config
|
||||||
|
Sidekiq.configure_server do |config|
|
||||||
|
config.redis = { url: ENV.fetch("REDIS_SERVER_URL", "redis://localhost:6379/0") }
|
||||||
|
end
|
||||||
|
|
||||||
|
schedule_file = "config/schedule.yml"
|
||||||
|
|
||||||
|
if File.exist?(schedule_file) && Sidekiq.server?
|
||||||
|
Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file)
|
||||||
|
end
|
||||||
|
|
||||||
|
# client config
|
||||||
|
Sidekiq.configure_client do |config|
|
||||||
|
config.redis = { url: ENV.fetch("REDIS_SERVER_URL", "redis://localhost:6379/0") }
|
||||||
|
end
|
||||||
14
config/initializers/wrap_parameters.rb
Normal file
14
config/initializers/wrap_parameters.rb
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Be sure to restart your server when you modify this file.
|
||||||
|
|
||||||
|
# This file contains settings for ActionController::ParamsWrapper which
|
||||||
|
# is enabled by default.
|
||||||
|
|
||||||
|
# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
|
||||||
|
ActiveSupport.on_load(:action_controller) do
|
||||||
|
wrap_parameters format: [:json]
|
||||||
|
end
|
||||||
|
|
||||||
|
# To enable root element in JSON for ActiveRecord objects.
|
||||||
|
# ActiveSupport.on_load(:active_record) do
|
||||||
|
# self.include_root_in_json = true
|
||||||
|
# end
|
||||||
9
config/local_env.yml
Normal file
9
config/local_env.yml
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Rename this file to local_env.yml
|
||||||
|
# Add account settings and API keys here.
|
||||||
|
# This file should be listed in .gitignore to keep your settings secret!
|
||||||
|
# Each entry gets set as a local environment variable.
|
||||||
|
# This file overrides ENV variables in the Unix shell.
|
||||||
|
# For example, setting:
|
||||||
|
# GMAIL_USERNAME: 'Your_Gmail_Username'
|
||||||
|
# makes 'Your_Gmail_Username' available as ENV["GMAIL_USERNAME"]
|
||||||
|
#GMAIL_USERNAME: 'Your_Gmail_Username'
|
||||||
33
config/locales/en.yml
Normal file
33
config/locales/en.yml
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
# Files in the config/locales directory are used for internationalization
|
||||||
|
# and are automatically loaded by Rails. If you want to use locales other
|
||||||
|
# than English, add the necessary files in this directory.
|
||||||
|
#
|
||||||
|
# To use the locales, use `I18n.t`:
|
||||||
|
#
|
||||||
|
# I18n.t 'hello'
|
||||||
|
#
|
||||||
|
# In views, this is aliased to just `t`:
|
||||||
|
#
|
||||||
|
# <%= t('hello') %>
|
||||||
|
#
|
||||||
|
# To use a different locale, set it with `I18n.locale`:
|
||||||
|
#
|
||||||
|
# I18n.locale = :es
|
||||||
|
#
|
||||||
|
# This would use the information in config/locales/es.yml.
|
||||||
|
#
|
||||||
|
# The following keys must be escaped otherwise they will not be retrieved by
|
||||||
|
# the default I18n backend:
|
||||||
|
#
|
||||||
|
# true, false, on, off, yes, no
|
||||||
|
#
|
||||||
|
# Instead, surround them with single quotes.
|
||||||
|
#
|
||||||
|
# en:
|
||||||
|
# 'true': 'foo'
|
||||||
|
#
|
||||||
|
# To learn more, please read the Rails Internationalization guide
|
||||||
|
# available at https://guides.rubyonrails.org/i18n.html.
|
||||||
|
|
||||||
|
en:
|
||||||
|
hello: "Hello world"
|
||||||
38
config/puma.rb
Normal file
38
config/puma.rb
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
# Puma can serve each request in a thread from an internal thread pool.
|
||||||
|
# The `threads` method setting takes two numbers: a minimum and maximum.
|
||||||
|
# Any libraries that use thread pools should be configured to match
|
||||||
|
# the maximum value specified for Puma. Default is set to 5 threads for minimum
|
||||||
|
# and maximum; this matches the default thread size of Active Record.
|
||||||
|
#
|
||||||
|
max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }
|
||||||
|
min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count }
|
||||||
|
threads min_threads_count, max_threads_count
|
||||||
|
|
||||||
|
# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
|
||||||
|
#
|
||||||
|
port ENV.fetch("PORT") { 3000 }
|
||||||
|
|
||||||
|
# Specifies the `environment` that Puma will run in.
|
||||||
|
#
|
||||||
|
environment ENV.fetch("RAILS_ENV") { "development" }
|
||||||
|
|
||||||
|
# Specifies the `pidfile` that Puma will use.
|
||||||
|
pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" }
|
||||||
|
|
||||||
|
# Specifies the number of `workers` to boot in clustered mode.
|
||||||
|
# Workers are forked web server processes. If using threads and workers together
|
||||||
|
# the concurrency of the application would be max `threads` * `workers`.
|
||||||
|
# Workers do not work on JRuby or Windows (both of which do not support
|
||||||
|
# processes).
|
||||||
|
#
|
||||||
|
# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
|
||||||
|
|
||||||
|
# Use the `preload_app!` method when specifying a `workers` number.
|
||||||
|
# This directive tells Puma to first boot the application and load code
|
||||||
|
# before forking the application. This takes advantage of Copy On Write
|
||||||
|
# process behavior so workers use less memory.
|
||||||
|
#
|
||||||
|
# preload_app!
|
||||||
|
|
||||||
|
# Allow puma to be restarted by `rails restart` command.
|
||||||
|
plugin :tmp_restart
|
||||||
9
config/routes.rb
Normal file
9
config/routes.rb
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
Rails.application.routes.draw do
|
||||||
|
require 'sidekiq/web'
|
||||||
|
mount Sidekiq::Web => '/sidekiq'
|
||||||
|
|
||||||
|
root to: 'pastes#index'
|
||||||
|
resources :pastes
|
||||||
|
|
||||||
|
get "search", to: "search#search"
|
||||||
|
end
|
||||||
5
config/schedule.yml
Normal file
5
config/schedule.yml
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
|
||||||
|
send_pastes_to_elastic_search_worker:
|
||||||
|
cron: "*/1 * * * *"
|
||||||
|
class: "SendPastesToElasticSearchWorker"
|
||||||
|
queue: send_pastes_to_elastic_search_worker
|
||||||
6
config/spring.rb
Normal file
6
config/spring.rb
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
Spring.watch(
|
||||||
|
".ruby-version",
|
||||||
|
".rbenv-vars",
|
||||||
|
"tmp/restart.txt",
|
||||||
|
"tmp/caching-dev.txt"
|
||||||
|
)
|
||||||
34
config/storage.yml
Normal file
34
config/storage.yml
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
test:
|
||||||
|
service: Disk
|
||||||
|
root: <%= Rails.root.join("tmp/storage") %>
|
||||||
|
|
||||||
|
local:
|
||||||
|
service: Disk
|
||||||
|
root: <%= Rails.root.join("storage") %>
|
||||||
|
|
||||||
|
# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
|
||||||
|
# amazon:
|
||||||
|
# service: S3
|
||||||
|
# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
|
||||||
|
# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
|
||||||
|
# region: us-east-1
|
||||||
|
# bucket: your_own_bucket
|
||||||
|
|
||||||
|
# Remember not to checkin your GCS keyfile to a repository
|
||||||
|
# google:
|
||||||
|
# service: GCS
|
||||||
|
# project: your_project
|
||||||
|
# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
|
||||||
|
# bucket: your_own_bucket
|
||||||
|
|
||||||
|
# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
|
||||||
|
# microsoft:
|
||||||
|
# service: AzureStorage
|
||||||
|
# storage_account_name: your_account_name
|
||||||
|
# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %>
|
||||||
|
# container: your_container_name
|
||||||
|
|
||||||
|
# mirror:
|
||||||
|
# service: Mirror
|
||||||
|
# primary: local
|
||||||
|
# mirrors: [ amazon, google, microsoft ]
|
||||||
5
config/webpack/development.js
Normal file
5
config/webpack/development.js
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
process.env.NODE_ENV = process.env.NODE_ENV || 'development'
|
||||||
|
|
||||||
|
const environment = require('./environment')
|
||||||
|
|
||||||
|
module.exports = environment.toWebpackConfig()
|
||||||
3
config/webpack/environment.js
Normal file
3
config/webpack/environment.js
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
const { environment } = require('@rails/webpacker')
|
||||||
|
|
||||||
|
module.exports = environment
|
||||||
5
config/webpack/production.js
Normal file
5
config/webpack/production.js
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
process.env.NODE_ENV = process.env.NODE_ENV || 'production'
|
||||||
|
|
||||||
|
const environment = require('./environment')
|
||||||
|
|
||||||
|
module.exports = environment.toWebpackConfig()
|
||||||
5
config/webpack/test.js
Normal file
5
config/webpack/test.js
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
process.env.NODE_ENV = process.env.NODE_ENV || 'development'
|
||||||
|
|
||||||
|
const environment = require('./environment')
|
||||||
|
|
||||||
|
module.exports = environment.toWebpackConfig()
|
||||||
95
config/webpacker.yml
Normal file
95
config/webpacker.yml
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
# Note: You must restart bin/webpack-dev-server for changes to take effect
|
||||||
|
|
||||||
|
default: &default
|
||||||
|
source_path: app/javascript
|
||||||
|
source_entry_path: packs
|
||||||
|
public_root_path: public
|
||||||
|
public_output_path: packs
|
||||||
|
cache_path: tmp/cache/webpacker
|
||||||
|
check_yarn_integrity: false
|
||||||
|
webpack_compile_output: false
|
||||||
|
|
||||||
|
# Additional paths webpack should lookup modules
|
||||||
|
# ['app/assets', 'engine/foo/app/assets']
|
||||||
|
resolved_paths: []
|
||||||
|
|
||||||
|
# Reload manifest.json on all requests so we reload latest compiled packs
|
||||||
|
cache_manifest: false
|
||||||
|
|
||||||
|
# Extract and emit a css file
|
||||||
|
extract_css: false
|
||||||
|
|
||||||
|
static_assets_extensions:
|
||||||
|
- .jpg
|
||||||
|
- .jpeg
|
||||||
|
- .png
|
||||||
|
- .gif
|
||||||
|
- .tiff
|
||||||
|
- .ico
|
||||||
|
- .svg
|
||||||
|
- .eot
|
||||||
|
- .otf
|
||||||
|
- .ttf
|
||||||
|
- .woff
|
||||||
|
- .woff2
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
- .mjs
|
||||||
|
- .js
|
||||||
|
- .sass
|
||||||
|
- .scss
|
||||||
|
- .css
|
||||||
|
- .module.sass
|
||||||
|
- .module.scss
|
||||||
|
- .module.css
|
||||||
|
- .png
|
||||||
|
- .svg
|
||||||
|
- .gif
|
||||||
|
- .jpeg
|
||||||
|
- .jpg
|
||||||
|
|
||||||
|
development:
|
||||||
|
<<: *default
|
||||||
|
compile: true
|
||||||
|
|
||||||
|
# Verifies that correct packages and versions are installed by inspecting package.json, yarn.lock, and node_modules
|
||||||
|
check_yarn_integrity: true
|
||||||
|
|
||||||
|
# Reference: https://webpack.js.org/configuration/dev-server/
|
||||||
|
dev_server:
|
||||||
|
https: false
|
||||||
|
host: localhost
|
||||||
|
port: 3035
|
||||||
|
public: localhost:3035
|
||||||
|
hmr: false
|
||||||
|
# Inline should be set to true if using HMR
|
||||||
|
inline: true
|
||||||
|
overlay: true
|
||||||
|
compress: true
|
||||||
|
disable_host_check: true
|
||||||
|
use_local_ip: false
|
||||||
|
quiet: false
|
||||||
|
headers:
|
||||||
|
'Access-Control-Allow-Origin': '*'
|
||||||
|
watch_options:
|
||||||
|
ignored: '**/node_modules/**'
|
||||||
|
|
||||||
|
|
||||||
|
test:
|
||||||
|
<<: *default
|
||||||
|
compile: true
|
||||||
|
|
||||||
|
# Compile test packs to a separate directory
|
||||||
|
public_output_path: packs-test
|
||||||
|
|
||||||
|
production:
|
||||||
|
<<: *default
|
||||||
|
|
||||||
|
# Production depends on precompilation of packs prior to booting for performance.
|
||||||
|
compile: false
|
||||||
|
|
||||||
|
# Extract and emit a css file
|
||||||
|
extract_css: true
|
||||||
|
|
||||||
|
# Cache manifest.json for performance
|
||||||
|
cache_manifest: true
|
||||||
9
db/migrate/20191023033047_create_pastes.rb
Normal file
9
db/migrate/20191023033047_create_pastes.rb
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
class CreatePastes < ActiveRecord::Migration[6.0]
|
||||||
|
def change
|
||||||
|
create_table :pastes do |t|
|
||||||
|
t.text :search_pastes
|
||||||
|
|
||||||
|
t.timestamps
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
21
db/schema.rb
Normal file
21
db/schema.rb
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
# This file is auto-generated from the current state of the database. Instead
|
||||||
|
# of editing this file, please use the migrations feature of Active Record to
|
||||||
|
# incrementally modify your database, and then regenerate this schema definition.
|
||||||
|
#
|
||||||
|
# This file is the source Rails uses to define your schema when running `rails
|
||||||
|
# db:schema:load`. When creating a new database, `rails db:schema:load` tends to
|
||||||
|
# be faster and is potentially less error prone than running all of your
|
||||||
|
# migrations from scratch. Old migrations may fail to apply correctly if those
|
||||||
|
# migrations use external dependencies or application code.
|
||||||
|
#
|
||||||
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
|
ActiveRecord::Schema.define(version: 2019_10_23_033047) do
|
||||||
|
|
||||||
|
create_table "pastes", force: :cascade do |t|
|
||||||
|
t.text "search_pastes"
|
||||||
|
t.datetime "created_at", precision: 6, null: false
|
||||||
|
t.datetime "updated_at", precision: 6, null: false
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
7
db/seeds.rb
Normal file
7
db/seeds.rb
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
# This file should contain all the record creation needed to seed the database with its default values.
|
||||||
|
# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
#
|
||||||
|
# movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
|
||||||
|
# Character.create(name: 'Luke', movie: movies.first)
|
||||||
69
docker-compose-2.4.yml
Normal file
69
docker-compose-2.4.yml
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
version: "2.4"
|
||||||
|
services:
|
||||||
|
web:
|
||||||
|
build: .
|
||||||
|
container_name: pastebinner-rails
|
||||||
|
ports:
|
||||||
|
- 3000:3000
|
||||||
|
depends_on:
|
||||||
|
- depends_on redis
|
||||||
|
- sidekiq
|
||||||
|
env_file:
|
||||||
|
- .pastebin_creds
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
redis:
|
||||||
|
image: redis:alpine
|
||||||
|
container_name: pastebinner-redis
|
||||||
|
ports:
|
||||||
|
- 6379:6379
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
sidekiq:
|
||||||
|
build: .
|
||||||
|
command: bundle exec sidekiq -c 3 -q "send_pastes_to_elastic_search_worker"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
elasticsearch:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
|
||||||
|
container_name: pastebinner-elasticsearch
|
||||||
|
environment:
|
||||||
|
- discovery.type=single-node
|
||||||
|
#- ES_JAVA_OPTS="-Xms1g -Xmx1g"
|
||||||
|
volumes:
|
||||||
|
- vibhuviesdata:/usr/share/elasticsearch/data
|
||||||
|
ports:
|
||||||
|
- 9200:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
labels:
|
||||||
|
- co.elastic.logs/module=elasticsearch
|
||||||
|
- co.elastic.metrics/module=elasticsearch
|
||||||
|
kibana:
|
||||||
|
image: docker.elastic.co/kibana/kibana:7.7.0
|
||||||
|
container_name: pastebinner-kibana
|
||||||
|
ports:
|
||||||
|
- 5601:5601
|
||||||
|
depends_on:
|
||||||
|
- elasticsearch
|
||||||
|
environment:
|
||||||
|
ELASTICSEARCH_URL: http://elasticsearch:9200
|
||||||
|
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
networks:
|
||||||
|
elastic:
|
||||||
|
driver: bridge
|
||||||
|
enable_ipv6: true
|
||||||
|
ipam:
|
||||||
|
driver: default
|
||||||
|
config:
|
||||||
|
- subnet: 2601:249:8b80:4020::/64
|
||||||
|
gateway: 2601:249:8b80:4020::1
|
||||||
|
volumes:
|
||||||
|
vibhuviesdata:
|
||||||
|
driver: local
|
||||||
63
docker-compose-3.9.yml
Normal file
63
docker-compose-3.9.yml
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
version: "3.9"
|
||||||
|
services:
|
||||||
|
web:
|
||||||
|
build: .
|
||||||
|
container_name: pastebinner-rails
|
||||||
|
ports:
|
||||||
|
- 3000:3000
|
||||||
|
depends_on:
|
||||||
|
- depends_on redis
|
||||||
|
- sidekiq
|
||||||
|
env_file:
|
||||||
|
- .pastebin_creds
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
redis:
|
||||||
|
image: redis:alpine
|
||||||
|
container_name: pastebinner-redis
|
||||||
|
ports:
|
||||||
|
- 6379:6379
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
sidekiq:
|
||||||
|
build: .
|
||||||
|
command: bundle exec sidekiq -c 3 -q "send_pastes_to_elastic_search_worker"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
elasticsearch:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
|
||||||
|
container_name: pastebinner-elasticsearch
|
||||||
|
environment:
|
||||||
|
- discovery.type=single-node
|
||||||
|
#- ES_JAVA_OPTS="-Xms1g -Xmx1g"
|
||||||
|
volumes:
|
||||||
|
- vibhuviesdata:/usr/share/elasticsearch/data
|
||||||
|
ports:
|
||||||
|
- 9200:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
labels:
|
||||||
|
- co.elastic.logs/module=elasticsearch
|
||||||
|
- co.elastic.metrics/module=elasticsearch
|
||||||
|
kibana:
|
||||||
|
image: docker.elastic.co/kibana/kibana:7.7.0
|
||||||
|
container_name: pastebinner-kibana
|
||||||
|
ports:
|
||||||
|
- 5601:5601
|
||||||
|
depends_on:
|
||||||
|
- elasticsearch
|
||||||
|
environment:
|
||||||
|
ELASTICSEARCH_URL: http://elasticsearch:9200
|
||||||
|
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
networks:
|
||||||
|
elastic:
|
||||||
|
external: true
|
||||||
|
volumes:
|
||||||
|
vibhuviesdata:
|
||||||
|
driver: local
|
||||||
70
docker-compose.yml
Normal file
70
docker-compose.yml
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
version: "2.4"
|
||||||
|
services:
|
||||||
|
redis:
|
||||||
|
image: redis:alpine
|
||||||
|
container_name: pastebinner-redis
|
||||||
|
ports:
|
||||||
|
- 6379:6379
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
web:
|
||||||
|
build: .
|
||||||
|
container_name: pastebinner-rails
|
||||||
|
ports:
|
||||||
|
- 3000:3000
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
- sidekiq
|
||||||
|
env_file:
|
||||||
|
- .pastebin_creds
|
||||||
|
- .env
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
sidekiq:
|
||||||
|
build: .
|
||||||
|
command: bundle exec sidekiq -c 3 -q "send_pastes_to_elastic_search_worker"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
elasticsearch:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
|
||||||
|
container_name: pastebinner-elasticsearch
|
||||||
|
environment:
|
||||||
|
- discovery.type=single-node
|
||||||
|
#- ES_JAVA_OPTS="-Xms1g -Xmx1g"
|
||||||
|
volumes:
|
||||||
|
- vibhuviesdata:/usr/share/elasticsearch/data
|
||||||
|
ports:
|
||||||
|
- 9200:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
labels:
|
||||||
|
- co.elastic.logs/module=elasticsearch
|
||||||
|
- co.elastic.metrics/module=elasticsearch
|
||||||
|
kibana:
|
||||||
|
image: docker.elastic.co/kibana/kibana:7.7.0
|
||||||
|
container_name: pastebinner-kibana
|
||||||
|
ports:
|
||||||
|
- 5601:5601
|
||||||
|
depends_on:
|
||||||
|
- elasticsearch
|
||||||
|
environment:
|
||||||
|
ELASTICSEARCH_URL: http://elasticsearch:9200
|
||||||
|
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||||
|
networks:
|
||||||
|
- elastic
|
||||||
|
networks:
|
||||||
|
elastic:
|
||||||
|
driver: bridge
|
||||||
|
enable_ipv6: true
|
||||||
|
ipam:
|
||||||
|
driver: default
|
||||||
|
config:
|
||||||
|
- subnet: 2601:249:8b80:4020::/64
|
||||||
|
gateway: 2601:249:8b80:4020::1
|
||||||
|
volumes:
|
||||||
|
vibhuviesdata:
|
||||||
|
driver: local
|
||||||
166
lib/api_client.rb
Normal file
166
lib/api_client.rb
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
class ApiClient
|
||||||
|
attr_accessor :api_dev_key, :username, :password
|
||||||
|
|
||||||
|
def initialize(api_dev_key, username, password)
|
||||||
|
@api_dev_key = api_dev_key
|
||||||
|
@username = username
|
||||||
|
@password = password
|
||||||
|
@base_api_url = 'https://pastebin.com/api'
|
||||||
|
@scraping_api_url = 'https://scrape.pastebin.com'
|
||||||
|
end
|
||||||
|
|
||||||
|
# this should be a hash of { endpoint_name: '/url_endpoint.php'}
|
||||||
|
ENDPOINTS = { login: '/api_login.php',
|
||||||
|
post: '/api_post.php',
|
||||||
|
raw: '/api_raw.php',
|
||||||
|
scraping: '/api_scraping.php',
|
||||||
|
scrape_item: '/api_scrape_item.php',
|
||||||
|
scrape_item_meta: '/api_scrape_item_meta.php' }.freeze
|
||||||
|
|
||||||
|
# basic example hash for creating a paste:
|
||||||
|
# params = { 'api_dev_key': @api_dev_key, 'api_option': 'paste'. 'api_paste_code': paste_data}
|
||||||
|
|
||||||
|
# required params:
|
||||||
|
# api_dev_key - your unique developer api key
|
||||||
|
# api_option - set as paste, this will indicate you want to create a new paste
|
||||||
|
# api_paste_code - this is the text that will be written inside of your paste
|
||||||
|
|
||||||
|
# optional params:
|
||||||
|
# api_user_key - this parameter is part of the login system, which is explained further down the page
|
||||||
|
# api_paste_name - this will be the name / title of your paste
|
||||||
|
# api_paste_format - this will be the syntax highlighting value, which is explained in detail further down the page
|
||||||
|
# api_paste_private - this makes a paste public, unlisted, or private, public = 0, unlisted = 1, private = 2
|
||||||
|
# api_paste_expire_date - this sets the expiration date of your paste, the values are explained further down the page
|
||||||
|
|
||||||
|
# example - params = { "api_dev_key": api_dev_key, "api_option": "paste", "api_paste_code": paste_data }
|
||||||
|
def create_paste(params)
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def api_user_key
|
||||||
|
# returns a user session key that can be used as the api_user_key param
|
||||||
|
@api_user_key ||= RestClient::Request.execute(
|
||||||
|
method: :post,
|
||||||
|
url: @base_api_url + ENDPOINTS[:login],
|
||||||
|
payload: { 'api_dev_key': @api_dev_key,
|
||||||
|
'api_user_name': @username,
|
||||||
|
'api_user_password': @password }
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def list_user_pastes
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_results_limit': '100',
|
||||||
|
'api_option': 'list' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def list_raw_user_paste(api_paste_key)
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_paste_key': api_paste_key,
|
||||||
|
'api_option': 'show_paste' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
# api_paste_key = this is the unique key of the paste data you want to delete.
|
||||||
|
def delete_user_paste(api_paste_key)
|
||||||
|
params = { 'api_dev_key': api_dev_key,
|
||||||
|
'api_user_key': api_user_key,
|
||||||
|
'api_paste_key': api_paste_key,
|
||||||
|
'api_option': 'delete' }
|
||||||
|
execute_query(:api_post, params)
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_user_info
|
||||||
|
params = { 'api_dev_key': api_dev_key }
|
||||||
|
end
|
||||||
|
|
||||||
|
def api_post(params)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :post,
|
||||||
|
url: @base_api_url + ENDPOINTS[:post],
|
||||||
|
payload: params
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
# params is optional for now. to query specific language ?lang=ruby as an example
|
||||||
|
# right now its set to grab the max 250, default is 50. param is ?limit=value
|
||||||
|
def scrape_public_pastes(_params = nil, limit)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scraping] + "?limit=#{limit}"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
# will extract just the keys from recent public pastes
|
||||||
|
def get_unique_paste_keys(public_pastes)
|
||||||
|
pp = JSON.parse(public_pastes)
|
||||||
|
pp.map { |p| p['key'] }
|
||||||
|
end
|
||||||
|
|
||||||
|
def raw_paste_data(unique_paste_key)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scrape_item] + "?i=#{unique_paste_key}"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def raw_paste_metadata(unique_paste_key)
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: @scraping_api_url + ENDPOINTS[:scrape_item_meta] + "?i=#{unique_paste_key}"
|
||||||
|
)
|
||||||
|
YAML.safe_load(response.body).first
|
||||||
|
end
|
||||||
|
|
||||||
|
def hash_pastes(keys)
|
||||||
|
if keys.is_a? String
|
||||||
|
raw_paste = self.raw_paste_data(keys)
|
||||||
|
raw_paste_metadata = self.raw_paste_metadata(keys)
|
||||||
|
hash = self.hash_paste(raw_paste, raw_paste_metadata)
|
||||||
|
else
|
||||||
|
keys.map do |key|
|
||||||
|
raw_paste = self.raw_paste_data(key).body
|
||||||
|
raw_paste_metadata = self.raw_paste_metadata(key)
|
||||||
|
hash = self.hash_paste(raw_paste, raw_paste_metadata)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def hash_paste(raw_paste_data, raw_paste_metadata)
|
||||||
|
{
|
||||||
|
"paste_metadata": raw_paste_metadata,
|
||||||
|
"paste_text": raw_paste_data
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_paste(key=nil, keys)
|
||||||
|
# if we give keys, create an array of X json pastes
|
||||||
|
if keys
|
||||||
|
self.hash_pastes(keys).map do |paste_hash|
|
||||||
|
paste_hash.to_json
|
||||||
|
end
|
||||||
|
else
|
||||||
|
# otherwise, just make a json of the 1 raw_paste_data & raw_paste_metadata
|
||||||
|
raw_paste_data = self.raw_paste_data(key)
|
||||||
|
raw_paste_metadata = self.raw_paste_metadata(key)
|
||||||
|
self.hash_paste(raw_paste_data, raw_paste_metadata).to_json
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_paste_from_file(raw_paste_json_file)
|
||||||
|
raw_paste_json = File.read(raw_paste_json_file)
|
||||||
|
self.hash_paste(raw_paste_json).to_json
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
# this will be the main way to execute any of these methods. this has the exception handling taken care of.
|
||||||
|
def execute_query(selector, *args)
|
||||||
|
send(selector, *args)
|
||||||
|
rescue RestClient::ExceptionWithResponse => e
|
||||||
|
puts e.message
|
||||||
|
end
|
||||||
|
end
|
||||||
0
lib/assets/.keep
Normal file
0
lib/assets/.keep
Normal file
80
lib/es_search_helper.rb
Normal file
80
lib/es_search_helper.rb
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
class EsSearchHelper
|
||||||
|
attr_accessor :server_uri, :index, :api_client, :doctype, :redis_connection
|
||||||
|
DEFAULT_METHOD = :post
|
||||||
|
|
||||||
|
def initialize(server_uri, index, doctype='paste')
|
||||||
|
@server_uri = server_uri
|
||||||
|
@index = index
|
||||||
|
@doctype = doctype
|
||||||
|
@api_client = ApiClient.new(ENV['pastebin_api_key'], ENV['pastebin_username'], ENV['pastebin_password'])
|
||||||
|
@redis_connection = RedisHelper.new(host: ENV['REDIS_SERVER_URL_PASTES_HOSTS'], db: ENV['REDIS_SERVER_URL_PASTES_DB'])
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_index
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :delete,
|
||||||
|
url: "#{server_uri}/#{index}")
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_mappings
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :get,
|
||||||
|
url: "#{server_uri}/#{index}/_mappings?pretty")
|
||||||
|
end
|
||||||
|
|
||||||
|
def create_mapping(mapping_json)
|
||||||
|
header = { 'Content-type': 'application/json' }
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :put,
|
||||||
|
url: "#{server_uri}/#{index}",
|
||||||
|
headers: header,
|
||||||
|
payload: mapping_json)
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_mapping(mapping_json)
|
||||||
|
header = { 'Content-type': 'application/json' }
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: :put,
|
||||||
|
url: "#{server_uri}/#{index}/_mapping/#{doctype}",
|
||||||
|
payload: mapping_json,
|
||||||
|
headers: header)
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_to_es(paste_json, method=nil)
|
||||||
|
header = { 'Content-type': 'application/json' }
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
method: method ||= DEFAULT_METHOD,
|
||||||
|
url: "#{server_uri}/#{index}/#{doctype}",
|
||||||
|
headers: header,
|
||||||
|
payload: paste_json)
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_to_es_bulk(array_of_paste_json)
|
||||||
|
array_of_paste_json.each do |paste_json|
|
||||||
|
self.json_to_es(paste_json)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def send_jsons_to_es(paste_max)
|
||||||
|
pastes = api_client.scrape_public_pastes(paste_max)
|
||||||
|
keys = api_client.get_unique_paste_keys(pastes)
|
||||||
|
keys_for_request = check_and_send_keys_redis(keys)
|
||||||
|
|
||||||
|
json_data = api_client.json_paste(keys_for_request)
|
||||||
|
self.json_to_es_bulk(json_data)
|
||||||
|
end
|
||||||
|
|
||||||
|
def check_and_send_keys_redis(keys)
|
||||||
|
keys_in_redis = keys.select do |key|
|
||||||
|
key if redis_connection.paste_key_exists?(key)
|
||||||
|
end
|
||||||
|
|
||||||
|
# we dont want to send a request if we already have the key present in redis.
|
||||||
|
keys_for_request = keys - keys_in_redis
|
||||||
|
|
||||||
|
keys_for_request.map do |key|
|
||||||
|
redis_connection.send_paste_key(key)
|
||||||
|
end
|
||||||
|
return keys_for_request
|
||||||
|
end
|
||||||
|
end
|
||||||
49
lib/github_gists.rb
Normal file
49
lib/github_gists.rb
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
require 'nokogiri'
|
||||||
|
require 'rest-client'
|
||||||
|
require 'pry'
|
||||||
|
|
||||||
|
class GithubGists
|
||||||
|
attr_accessor :gist_base_url, :discover_url
|
||||||
|
|
||||||
|
def initialize()
|
||||||
|
@gist_base_url = 'https://gist.github.com'
|
||||||
|
@discover_url = "#{gist_base_url}/discover"
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_gists_from_discover
|
||||||
|
# there are some url params we can use to fine tweak this.
|
||||||
|
# - recently created
|
||||||
|
# - last recently created
|
||||||
|
# - recently updated
|
||||||
|
# - last recently updated
|
||||||
|
# we can add these params in instead of just one fetch of the regular discover_url
|
||||||
|
response = RestClient::Request.execute(
|
||||||
|
:method => :get,
|
||||||
|
:url => @discover_url
|
||||||
|
)
|
||||||
|
|
||||||
|
if response.code == 200
|
||||||
|
html = Nokogiri::HTML(response.body)
|
||||||
|
else
|
||||||
|
"Http Status: #{response.code}"
|
||||||
|
end
|
||||||
|
# this xpath should get us an array of urls of user gists: example: https://gist.github.com/joberding/3e408b8ec7ddb932ebef1b5b94a37255
|
||||||
|
html.xpath('//*[@class="link-overlay"]').map {|node| node.values[1]}
|
||||||
|
end
|
||||||
|
|
||||||
|
def get_raw_gist_from_url(url)
|
||||||
|
# example input url: https://gist.github.com/joberding/3e408b8ec7ddb932ebef1b5b94a37255
|
||||||
|
# example output url: https://gist.githubusercontent.com/joberding/3e408b8ec7ddb932ebef1b5b94a37255/raw
|
||||||
|
end
|
||||||
|
|
||||||
|
def available_url_params
|
||||||
|
{
|
||||||
|
:recently_created => '?direction=desc&sort=created',
|
||||||
|
:least_recently_created => '?direction=asc&sort=created',
|
||||||
|
:recently_updated => '?direction=desc&sort=updated',
|
||||||
|
:least_recently_updated => '?direction=asc&sort=updated'
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
end
|
||||||
17
lib/redis_helper.rb
Normal file
17
lib/redis_helper.rb
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
class RedisHelper
|
||||||
|
attr_accessor :connection
|
||||||
|
|
||||||
|
def initialize(host: host, db: db)
|
||||||
|
@host = host
|
||||||
|
@db = db
|
||||||
|
@connection = Redis.new(host: host, db: db)
|
||||||
|
end
|
||||||
|
|
||||||
|
def send_paste_key(key)
|
||||||
|
connection.sadd(key, 'pastes')
|
||||||
|
end
|
||||||
|
|
||||||
|
def paste_key_exists?(key)
|
||||||
|
connection.sismember(key, 'pastes')
|
||||||
|
end
|
||||||
|
end
|
||||||
0
lib/tasks/.keep
Normal file
0
lib/tasks/.keep
Normal file
0
log/.keep
Normal file
0
log/.keep
Normal file
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue