Merge pull request #4700 from pulibrary/sidekiq-email

Use sidekiq as email delivery queue

.lando.yml

@@ -22,7 +22,7 @@ services:
     type: postgres:15
     portforward: true
   orangelight_redis:
-    type: redis:6.0.16
+    type: redis:7
     portforward: true
 proxy:
   orangelight_test_solr:

Gemfile

@@ -76,6 +76,7 @@ gem 'rsolr'
 # Should this be in the test, development group?
 gem 'rspec-rails'
 gem 'rubyzip', '>= 1.2.2'
+gem 'sidekiq'
 gem 'simple_form'
 gem 'sneakers'
 gem 'sprockets-es6'

Gemfile.lock

@@ -615,6 +615,11 @@ GEM
     serverengine (2.0.7)
       sigdump (~> 0.2.2)
     set (1.1.1)
+    sidekiq (7.3.7)
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
     sigdump (0.2.5)
     simple_form (5.3.1)
       actionpack (>= 5.2)
@@ -826,6 +831,7 @@ DEPENDENCIES
   rubocop-rspec_rails
   rubyzip (>= 1.2.2)
   selenium-webdriver
+  sidekiq
   simple_form
   sneakers
   solargraph

README.md

@@ -56,15 +56,17 @@ bundle exec cap staging deploy # deploys main branch to staging
 ```
 
 ## Staging Mail Catcher
-  To see mail that has been sent on the staging server you must ssh tunnel into the server
+  To see mail that has been sent from the staging environment you must ssh tunnel into the staging indexer servers.
   ```
-  ssh -L 1082:localhost:1080 pulsys@catalog-staging1
+  ssh -L 1082:localhost:1080 pulsys@catalog-indexer-staging1
+  ssh -L 1085:localhost:1080 pulsys@catalog-indexer-staging2
   ```
   To see mail that has been sent on the qa server you must ssh tunnel into the server
   ```
-  ssh -L 1082:localhost:1080 pulsys@catalog-qa1
+  ssh -L 1082:localhost:1080 pulsys@catalog-indexer-qa1
+  ssh -L 1085:localhost:1080 pulsys@catalog-indexer-qa2
   ```
-  Once the tunnel is open [you can see the mail that has been sent on staging here]( http://localhost:1082/)
+  Once the tunnel is open [you can see the mail that has been sent on indexer1 here]( http://localhost:1082/) and [indexer2 here]( http://localhost:1085/)
 
 ## Testing
 
@@ -245,6 +247,7 @@ Start the Bibdata server, and then set the ```bidata_base``` value in OrangeLigh
      ```
      run every time
      ```
+     bundle exec sidekiq -q default -q mailers
      mailcatcher
      ```
 

config/application.rb

@@ -50,6 +50,8 @@ class Application < Rails::Application
     # config.time_zone = 'Central Time (US & Canada)'
     config.time_zone = 'Eastern Time (US & Canada)'
 
+    config.active_job.queue_adapter = :sidekiq
+
     # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     # config.i18n.default_locale = :de

config/deploy.rb

@@ -8,8 +8,6 @@
 
 set :branch, ENV.fetch('BRANCH', 'main')
 
-# Default branch is :master
-# ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }.call
 
 # Default deploy_to directory is /var/www/my_app
 set :deploy_to, '/opt/orangelight'
@@ -159,5 +157,15 @@
   end
 end
 
+namespace :sidekiq do
+  task :restart do
+    on roles(:indexer) do
+      execute :sudo, :service, :sidekiq, :restart
+    end
+  end
+end
+
 after 'deploy:reverted', 'sneakers:restart'
 after 'deploy:published', 'sneakers:restart'
+after 'deploy:reverted', 'sidekiq:restart'
+after 'deploy:published', 'sidekiq:restart'

config/deploy/production.rb

@@ -18,9 +18,9 @@
 server 'catalog3', user: 'deploy', roles: %i[web app db worker]
 server 'catalog4', user: 'deploy', roles: %i[web app db worker]
 server 'catalog5', user: 'deploy', roles: %i[web app db worker]
-server 'catalog-indexer1', user: 'deploy', roles: %i[cron_prod1 cron_db worker]
-server 'catalog-indexer2', user: 'deploy', roles: %i[cron_prod2 worker]
-server 'catalog-indexer3', user: 'deploy', roles: %i[cron_prod3 worker]
+server 'catalog-indexer1', user: 'deploy', roles: %i[cron_prod1 cron_db worker indexer]
+server 'catalog-indexer2', user: 'deploy', roles: %i[cron_prod2 worker indexer]
+server 'catalog-indexer3', user: 'deploy', roles: %i[cron_prod3 worker indexer]
 
 set :deploy_to, '/opt/orangelight'
 set :log_level, :info

config/deploy/qa.rb

@@ -6,8 +6,8 @@
 
 server 'catalog-qa1.princeton.edu', user: 'deploy', roles: %i[web app db worker mailcatcher]
 server 'catalog-qa2.princeton.edu', user: 'deploy', roles: %i[web app db worker mailcatcher cron_db]
-server 'catalog-indexer-qa1', user: 'deploy', roles: %i[worker]
-server 'catalog-indexer-qa2', user: 'deploy', roles: %i[cron_db worker]
+server 'catalog-indexer-qa1', user: 'deploy', roles: %i[worker indexer]
+server 'catalog-indexer-qa2', user: 'deploy', roles: %i[cron_db worker indexer]
 
 namespace :env do
   desc 'Set an Orangelight environment variable'

config/deploy/staging.rb

@@ -6,8 +6,8 @@
 
 server 'catalog-staging1', user: 'deploy', roles: %i[web app db worker mailcatcher cron_db]
 server 'catalog-staging2', user: 'deploy', roles: %i[web app worker mailcatcher]
-server 'catalog-indexer-staging1', user: 'deploy', roles: %i[cron_db worker]
-server 'catalog-indexer-staging2', user: 'deploy', roles: %i[cron_db worker]
+server 'catalog-indexer-staging1', user: 'deploy', roles: %i[cron_db worker indexer]
+server 'catalog-indexer-staging2', user: 'deploy', roles: %i[cron_db worker indexer]
 
 namespace :env do
   desc 'Set an Orangelight environment variable'

config/initializers/sidekiq.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "redis-client"
+
+# nosemgrep
+redis_config_from_yml = YAML.safe_load(ERB.new(Rails.root.join("config/redis.yml").read).result, aliases: true)[Rails.env].with_indifferent_access
+redis_config = RedisClient.config(host: redis_config_from_yml[:host], port: redis_config_from_yml[:port], db: redis_config_from_yml[:db])
+
+# Sidekiq::Client.reliable_push! unless Rails.env.test? # sidekiq pro feature, we would need the key and sidekiq pro gem to enable this
+Sidekiq.configure_server do |config|
+  config.redis = { url: redis_config.server_url }
+  # config.super_fetch! # sidekiq pro feature
+  # config.reliable_scheduler! # sidekiq pro feature
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: redis_config.server_url }
+end

config/routes.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq/web'
+
 Rails.application.routes.draw do
   scope module: 'orangelight' do
     get 'browse', to: 'browsables#browse'
@@ -65,6 +67,10 @@
     get "sign_out", to: "sessions#destroy"
   end
 
+  authenticate :user, ->(user) { user.admin? } do
+    mount Sidekiq::Web => '/sidekiq'
+  end
+
   get '/catalog/oclc/:id', to: 'catalog#oclc'
   get '/catalog/isbn/:id', to: 'catalog#isbn'
   get '/catalog/lccn/:id', to: 'catalog#lccn'

spec/requests/sidekiq_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+require 'rails_helper'
+RSpec.describe 'Sidekiq UI' do
+  it 'denies access to the sidekiq UI to non-admins' do
+    user = FactoryBot.create(:user)
+    login_as user
+    expect do
+      visit '/sidekiq'
+    end.to raise_error ActionController::RoutingError
+  end
+  it 'allows access to the sidekiq UI to admins' do
+    user = FactoryBot.create(:user)
+    allow(user).to receive(:admin?).and_return true
+    login_as user
+    visit '/sidekiq'
+    expect(page).to have_text 'Sidekiq'
+  end
+end