diff --git a/config/sidekiq.service b/config/sidekiq.service new file mode 100644 index 0000000..8e6c925 --- /dev/null +++ b/config/sidekiq.service @@ -0,0 +1,52 @@ +# +# systemd unit file for CentOS 7, Ubuntu 15.04 +# +# Customize this file based on your bundler location, app directory, etc. +# Put this in /usr/lib/systemd/system (CentOS) or /lib/systemd/system (Ubuntu). +# Run: +# - systemctl enable sidekiq +# - systemctl {start,stop,restart} sidekiq +# +# This file corresponds to a single Sidekiq process. Add multiple copies +# to run multiple processes (sidekiq-1, sidekiq-2, etc). +# +# See Inspeqtor's Systemd wiki page for more detail about Systemd: +# https://github.com/mperham/inspeqtor/wiki/Systemd +# +[Unit] +Description=sidekiq +# start us only once the network and logging subsystems are available, +# consider adding redis-server.service if Redis is local and systemd-managed. +After=syslog.target network.target + +# See these pages for lots of options: +# http://0pointer.de/public/systemd-man/systemd.service.html +# http://0pointer.de/public/systemd-man/systemd.exec.html +[Service] +Type=simple +WorkingDirectory=/opt/myapp/current +# If you use rbenv: +# ExecStart=/bin/bash -lc '/home/deploy/.rbenv/shims/bundle exec sidekiq -e production' +# If you use the system's ruby: +ExecStart=/usr/local/bin/bundle exec sidekiq -e production +User=deploy +Group=deploy +UMask=0002 + +# Greatly reduce Ruby memory fragmentation and heap usage +# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/ +Environment=MALLOC_ARENA_MAX=2 + +# if we crash, restart +RestartSec=1 +Restart=on-failure + +# output goes to /var/log/syslog +StandardOutput=syslog +StandardError=syslog + +# This will default to "bundler" if we don't specify it +SyslogIdentifier=sidekiq + +[Install] +WantedBy=multi-user.target diff --git a/helpers/elastic_search_helper.rb b/helpers/elastic_search_helper.rb index 4b3957b..68dc6ba 100644 --- a/helpers/elastic_search_helper.rb +++ b/helpers/elastic_search_helper.rb @@ -44,5 +44,4 @@ class ElasticSearchHelper self.json_to_es(paste_json) end end - end diff --git a/helpers/sidekiq_helper.rb b/helpers/sidekiq_helper.rb new file mode 100644 index 0000000..5865378 --- /dev/null +++ b/helpers/sidekiq_helper.rb @@ -0,0 +1,12 @@ +require 'sidekiq' +require 'sidekiq/api' + +# load up the redis cfg +require '../config/initializers/sidekiq' + +class SidekiqHelper + + def initialize + end + +end diff --git a/lib/pastebinner.rb b/lib/pastebinner.rb index be9ea00..904432e 100755 --- a/lib/pastebinner.rb +++ b/lib/pastebinner.rb @@ -169,6 +169,11 @@ class Pastebinner end end + def json_paste_from_file(raw_paste_json_file) + raw_paste_json = File.read(raw_paste_json_file) + self.hash_paste(raw_paste_json).to_json + end + # keep this method private so we are not letting anyone run any method in our program private diff --git a/workers/paste_to_es.rb b/workers/paste_to_es.rb index a01acf8..94b5daf 100644 --- a/workers/paste_to_es.rb +++ b/workers/paste_to_es.rb @@ -1,9 +1,14 @@ class PasteToEs include Sidekiq::Worker + + sidekiq_options retry: false # i dont want to get rate limited so im just letting this fail if their are any failures def perform(es_object, pb_object, paste_max) Logger.new(STDOUT).info("PasteToEs started") + # get public pastes and their keys pastes = pb_object.scrape_public_pastes(paste_max) keys = pb_object.get_unique_paste_keys(pastes) + + # build it into json and send it to elasticsearch json_data = pb_object.json_paste(keys) es_object.json_to_es_bulk(json_data) end