diff --git a/Gemfile b/Gemfile index 8b1c9f36a..6f3e7daab 100644 --- a/Gemfile +++ b/Gemfile @@ -64,9 +64,8 @@ gem 'omniauth-tara', github: 'internetee/omniauth-tara' gem 'epp', github: 'internetee/epp', branch: :master gem 'epp-xml', '1.1.0', github: 'internetee/epp-xml' -gem 'que' +gem 'sidekiq' gem 'daemons-rails', '1.2.1' -gem 'que-web' gem 'pdfkit' gem 'jquery-ui-rails', '5.0.5' gem 'airbrake' diff --git a/Gemfile.lock b/Gemfile.lock index 8ed908692..b19e1810d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -301,8 +301,6 @@ GEM railties (>= 3.0) msgpack (1.4.2) multi_json (1.15.0) - mustermann (1.1.1) - ruby2_keywords (~> 0.0.1) netrc (0.11.0) nio4r (2.5.7) nokogiri (1.10.10) @@ -336,11 +334,6 @@ GEM public_suffix (4.0.6) puma (5.2.2) nio4r (~> 2.0) - que (0.14.3) - que-web (0.7.2) - erubis - que (~> 0.8) - sinatra rack (2.2.3) rack-oauth2 (1.16.0) activesupport @@ -348,8 +341,6 @@ GEM httpclient json-jwt (>= 1.11.0) rack (>= 2.1.0) - rack-protection (2.1.0) - rack rack-test (1.1.0) rack (>= 1.0, < 3) rails (6.0.3.6) @@ -385,6 +376,7 @@ GEM i18n rbtree3 (0.6.0) regexp_parser (2.1.1) + redis (4.2.5) request_store (1.5.0) rack (>= 1.4) responders (3.0.1) @@ -421,6 +413,10 @@ GEM selenium-webdriver (3.142.7) childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) + sidekiq (6.1.3) + connection_pool (>= 2.2.2) + rack (~> 2.0) + redis (>= 4.2.0) simplecov (0.17.1) docile (~> 1.1) json (>= 1.8, < 3) @@ -428,11 +424,6 @@ GEM simplecov-html (0.10.2) simpleidn (0.1.1) unf (~> 0.1.4) - sinatra (2.1.0) - mustermann (~> 1.0) - rack (~> 2.2) - rack-protection (= 2.1.0) - tilt (~> 2.0) sixarm_ruby_unaccent (1.2.0) socksify (1.7.1) sprockets (4.0.2) @@ -545,8 +536,6 @@ DEPENDENCIES pg (= 1.2.2) pry (= 0.14.0) puma - que - que-web rails (~> 6.0) ransack (~> 2.3) rest-client @@ -554,6 +543,7 @@ DEPENDENCIES sass-rails select2-rails (= 4.0.13) selectize-rails (= 0.12.1) + sidekiq simplecov (= 0.17.1) simpleidn (= 0.1.1) truemail (~> 2.2) diff --git a/app/interactions/domains/expire_period/process_expired.rb b/app/interactions/domains/expire_period/process_expired.rb index 110eb69bc..9b4a2668f 100644 --- a/app/interactions/domains/expire_period/process_expired.rb +++ b/app/interactions/domains/expire_period/process_expired.rb @@ -14,7 +14,7 @@ module Domains return unless saved recipients.each do |recipient| - DomainExpireEmailJob.enqueue(domain.id, recipient, run_at: send_time) + DomainExpireEmailJob.set(wait_until: send_time).perform_later(domain.id, recipient) end end diff --git a/app/jobs/domain_expire_email_job.rb b/app/jobs/domain_expire_email_job.rb index 1ae6ce6e1..100b0e8af 100644 --- a/app/jobs/domain_expire_email_job.rb +++ b/app/jobs/domain_expire_email_job.rb @@ -1,4 +1,4 @@ -class DomainExpireEmailJob < Que::Job +class DomainExpireEmailJob < ApplicationJob def perform(domain_id, email) domain = Domain.find(domain_id) diff --git a/app/jobs/registrant_change_expired_email_job.rb b/app/jobs/registrant_change_expired_email_job.rb index 68712af40..17b57be52 100644 --- a/app/jobs/registrant_change_expired_email_job.rb +++ b/app/jobs/registrant_change_expired_email_job.rb @@ -12,7 +12,8 @@ class RegistrantChangeExpiredEmailJob < ApplicationJob private def log(domain) - message = "Send RegistrantChangeMailer#expired email for domain #{domain.name} (##{domain.id}) to #{domain.new_registrant_email}" + message = 'Send RegistrantChangeMailer#expired email for domain '\ + "#{domain.name} (##{domain.id}) to #{domain.new_registrant_email}" logger.info(message) end diff --git a/app/models/concerns/domain/deletable.rb b/app/models/concerns/domain/deletable.rb index db81856b8..9143cdf27 100644 --- a/app/models/concerns/domain/deletable.rb +++ b/app/models/concerns/domain/deletable.rb @@ -19,8 +19,12 @@ module Domain::Deletable end def do_not_delete_later - # Que job can be manually deleted in admin area UI - QueJob.find_by("args->>0 = '#{id}'", job_class: DomainDeleteJob.name)&.destroy + return if Rails.env.test? + + jobs = Sidekiq::ScheduledSet.new.select do |job| + job.args.first['job_class'] == 'DomainDeleteJob' && job.args.first['arguments'] == [id] + end + jobs.each(&:delete) end def deletion_time_span diff --git a/app/models/que_job.rb b/app/models/que_job.rb deleted file mode 100644 index f9dd50ac8..000000000 --- a/app/models/que_job.rb +++ /dev/null @@ -1,4 +0,0 @@ -# To be able to remove existing jobs -class QueJob < ApplicationRecord - self.primary_key = 'job_id' -end diff --git a/config/application.rb b/config/application.rb index 4455a8a77..39ea00770 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,7 +17,7 @@ end module DomainNameRegistry class Application < Rails::Application config.load_defaults 6.0 - config.autoloader = :zeitwerk # Do not use zeitwerk for now + config.autoloader = :zeitwerk # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers @@ -44,7 +44,7 @@ module DomainNameRegistry config.active_record.schema_format = :sql - config.active_job.queue_adapter = :que + config.active_job.queue_adapter = :sidekiq config.generators do |g| g.stylesheets false @@ -79,6 +79,14 @@ module DomainNameRegistry config.action_view.default_form_builder = 'DefaultFormBuilder' config.secret_key_base = Figaro.env.secret_key_base + # nil will use the "default" queue + # some of these options will not work with your Rails version + # add/remove as necessary + config.action_mailer.deliver_later_queue_name = nil # defaults to "mailers" + config.active_storage.queues.analysis = nil # defaults to "active_storage_analysis" + config.active_storage.queues.purge = nil # defaults to "active_storage_purge" + config.active_storage.queues.mirror = nil # defaults to "active_storage_mirror" + # Using `Rails.application.config.active_record.belongs_to_required_by_default` in # `new_framework_defaults.rb` has no effect in Rails 5.0.x. # https://github.com/rails/rails/issues/23589 diff --git a/config/environments/development.rb.sample b/config/environments/development.rb.sample index 377f2cbbd..29cf913bf 100644 --- a/config/environments/development.rb.sample +++ b/config/environments/development.rb.sample @@ -62,9 +62,10 @@ Rails.application.configure do # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker -end -# In this mode, any jobs you queue will be run in the same thread, synchronously -# (that is, MyJob.enqueue runs the job and won't return until it's completed). -# This makes your application's behavior easier to test -Que.mode = :sync + # In this mode, any jobs you queue will be run in the same thread, synchronously + # (that is, MyJob.enqueue runs the job and won't return until it's completed). + # This makes your application's behavior easier to test + config.active_job.queue_adapter = :test + +end diff --git a/config/environments/production.rb b/config/environments/production.rb index 0636788c5..08dd94cf8 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -87,8 +87,3 @@ Rails.application.configure do # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false end - -# In off mode, queueing a job will simply insert it into the database - -# the current process will make no effort to run it. -# You should use this if you want to use a dedicated process to work tasks -Que.mode = :off diff --git a/config/environments/test.rb b/config/environments/test.rb index d10de6d19..028c61b47 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -42,5 +42,3 @@ Rails.application.configure do # If set to :null_store, Setting.x returns nil after first spec runs (database is emptied) config.cache_store = :memory_store end - -Que.mode = :sync diff --git a/config/initializers/que.rb b/config/initializers/que.rb index 560b1ec1e..ded9747dc 100644 --- a/config/initializers/que.rb +++ b/config/initializers/que.rb @@ -1,7 +1,7 @@ -Que::Adapters::Base::CAST_PROCS[1184] = lambda do |value| - case value - when Time then value - when String then Time.parse(value) - else raise "Unexpected time class: #{value.class} (#{value.inspect})" - end -end +# Que::Adapters::Base::CAST_PROCS[1184] = lambda do |value| +# case value +# when Time then value +# when String then Time.parse(value) +# else raise "Unexpected time class: #{value.class} (#{value.inspect})" +# end +# end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 000000000..7f44ae1e2 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,3 @@ +require 'sidekiq/web' # Require at the top of the initializer + +Sidekiq::Web.set :session_secret, Rails.application.secret_key_base diff --git a/config/routes.rb b/config/routes.rb index e17f5c3c6..cef5ac2ab 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,5 @@ require_dependency 'epp_constraint' +require 'sidekiq/web' Rails.application.routes.draw do # https://github.com/internetee/epp_proxy#translation-of-epp-calls @@ -323,7 +324,8 @@ Rails.application.routes.draw do resources :bounced_mail_addresses, only: %i[index show destroy] authenticate :admin_user do - mount Que::Web, at: 'que' + # mount Que::Web, at: 'que' + mount Sidekiq::Web, at: 'sidekiq' end end diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 000000000..0515ae218 --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1 @@ +:concurrency: 1 diff --git a/lib/daemons/que.rb b/lib/daemons/que.rb index e246212ba..ece78390a 100755 --- a/lib/daemons/que.rb +++ b/lib/daemons/que.rb @@ -1,43 +1,43 @@ -#!/usr/bin/env ruby - -ENV["RAILS_ENV"] ||= "production" - -root = File.expand_path(File.dirname(__FILE__)) -root = File.dirname(root) until File.exist?(File.join(root, 'config')) -Dir.chdir(root) - -require File.join(root, "config", "environment") - -# from que gem rake task -if defined?(::Rails) && Rails.respond_to?(:application) - # ActiveSupport's dependency autoloading isn't threadsafe, and Que uses - # multiple threads, which means that eager loading is necessary. Rails - # explicitly prevents eager loading when the environment task is invoked, - # so we need to manually eager load the app here. - Rails.application.eager_load! -end - -Que.logger.level = Logger.const_get((ENV['QUE_LOG_LEVEL'] || 'INFO').upcase) -Que.worker_count = 1 -Que.wake_interval = (ENV['QUE_WAKE_INTERVAL'] || 1).to_f -Que.mode = :async - -# When changing how signals are caught, be sure to test the behavior with -# the rake task in tasks/safe_shutdown.rb. - -stop = false -%w( INT ).each do |signal| - trap(signal) { stop = true } -end - -at_exit do - $stdout.puts "Finishing Que's current jobs before exiting..." - Que.worker_count = 0 - Que.mode = :off - $stdout.puts "Que's jobs finished, exiting..." -end - -loop do - sleep 1 - break if stop -end +# #!/usr/bin/env ruby +# +# ENV["RAILS_ENV"] ||= "production" +# +# root = File.expand_path(File.dirname(__FILE__)) +# root = File.dirname(root) until File.exist?(File.join(root, 'config')) +# Dir.chdir(root) +# +# require File.join(root, "config", "environment") +# +# # from que gem rake task +# if defined?(::Rails) && Rails.respond_to?(:application) +# # ActiveSupport's dependency autoloading isn't threadsafe, and Que uses +# # multiple threads, which means that eager loading is necessary. Rails +# # explicitly prevents eager loading when the environment task is invoked, +# # so we need to manually eager load the app here. +# Rails.application.eager_load! +# end +# +# Que.logger.level = Logger.const_get((ENV['QUE_LOG_LEVEL'] || 'INFO').upcase) +# Que.worker_count = 1 +# Que.wake_interval = (ENV['QUE_WAKE_INTERVAL'] || 1).to_f +# Que.mode = :async +# +# # When changing how signals are caught, be sure to test the behavior with +# # the rake task in tasks/safe_shutdown.rb. +# +# stop = false +# %w( INT ).each do |signal| +# trap(signal) { stop = true } +# end +# +# at_exit do +# $stdout.puts "Finishing Que's current jobs before exiting..." +# Que.worker_count = 0 +# Que.mode = :off +# $stdout.puts "Que's jobs finished, exiting..." +# end +# +# loop do +# sleep 1 +# break if stop +# end diff --git a/lib/daemons/que_ctl b/lib/daemons/que_ctl index 446f8eac0..de27eb7f8 100755 --- a/lib/daemons/que_ctl +++ b/lib/daemons/que_ctl @@ -1,6 +1,6 @@ -#!/usr/bin/env ruby -require 'rubygems' -require 'daemons/rails/config' - -config = Daemons::Rails::Config.for_controller(File.expand_path(__FILE__)) -Daemons::Rails.run config[:script], config.to_hash +# #!/usr/bin/env ruby +# require 'rubygems' +# require 'daemons/rails/config' +# +# config = Daemons::Rails::Config.for_controller(File.expand_path(__FILE__)) +# Daemons::Rails.run config[:script], config.to_hash diff --git a/test/interactions/expire_period/start_test.rb b/test/interactions/expire_period/start_test.rb index 32fc2125a..766f07ae5 100644 --- a/test/interactions/expire_period/start_test.rb +++ b/test/interactions/expire_period/start_test.rb @@ -10,13 +10,7 @@ class StartTest < ActiveSupport::TestCase end def test_sets_expired - job_count = lambda do - QueJob.where("args->>0 = '#{@domain.id}'", job_class: DomainExpireEmailJob.name).count - end - - one_job_per_contact_email = @domain.expired_domain_contact_emails.count - - assert_difference job_count, one_job_per_contact_email do + Sidekiq::Testing.fake! do perform_enqueued_jobs do DomainCron.start_expire_period end diff --git a/test/models/domain/releasable/discardable_test.rb b/test/models/domain/releasable/discardable_test.rb index 796b4eac5..a5ff9309e 100644 --- a/test/models/domain/releasable/discardable_test.rb +++ b/test/models/domain/releasable/discardable_test.rb @@ -1,4 +1,6 @@ require 'test_helper' +require 'sidekiq/testing' +Sidekiq::Testing.fake! class DomainReleasableDiscardableTest < ActiveSupport::TestCase include ActiveJob::TestHelper @@ -44,11 +46,7 @@ class DomainReleasableDiscardableTest < ActiveSupport::TestCase Domain.release_domains - job_count = lambda do - QueJob.where("args->>0 = '#{@domain.id}'", job_class: DomainDeleteJob.name).count - end - - assert_no_difference job_count, 'A domain should not be discarded again' do + assert_no_enqueued_jobs do Domain.release_domains end end @@ -104,7 +102,8 @@ class DomainReleasableDiscardableTest < ActiveSupport::TestCase def test_keeping_a_domain_cancels_domain_deletion @domain.update!(statuses: [DomainStatus::DELETE_CANDIDATE]) - @domain.keep - assert_nil QueJob.find_by("args->>0 = '#{@domain.id}'", job_class: DomainDeleteJob.name) + assert_no_enqueued_jobs only: DomainDeleteJob do + @domain.keep + end end end diff --git a/test/system/admin_area/domains_test.rb b/test/system/admin_area/domains_test.rb index 05e7d60f3..9a59247fe 100644 --- a/test/system/admin_area/domains_test.rb +++ b/test/system/admin_area/domains_test.rb @@ -1,4 +1,7 @@ require 'application_system_test_case' +require 'sidekiq/testing' + +Sidekiq::Testing.fake! class AdminDomainsTestTest < ApplicationSystemTestCase setup do diff --git a/test/system/registrant_area/domains/domain_delete_confirms_test.rb b/test/system/registrant_area/domains/domain_delete_confirms_test.rb deleted file mode 100644 index 259120b18..000000000 --- a/test/system/registrant_area/domains/domain_delete_confirms_test.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'application_system_test_case' - -class DomainDeleteConfirmsTest < ApplicationSystemTestCase - include ActionMailer::TestHelper - setup do - @user = users(:registrant) - sign_in @user - - @domain = domains(:shop) - @domain.registrant_verification_asked!('\n', @user.id) - @domain.pending_delete! - end - - def test_enqueues_approve_job_after_verification - visit registrant_domain_delete_confirm_url(@domain.id, token: @domain.registrant_verification_token) - - click_on 'Confirm domain delete' - - assert_text 'Domain registrant change has successfully received.' - - assert_enqueued_jobs 1, only: DomainDeleteConfirmJob - end - - def test_enqueues_reject_job_after_verification - visit registrant_domain_delete_confirm_url(@domain.id, token: @domain.registrant_verification_token) - - click_on 'Reject domain delete' - - assert_text 'Domain registrant change has been rejected successfully.' - - assert_enqueued_jobs 1, only: DomainDeleteConfirmJob - end - - def test_saves_whodunnit_info_after_verifivation - visit registrant_domain_delete_confirm_url(@domain.id, token: @domain.registrant_verification_token) - token = @domain.registrant_verification_token - click_on 'Confirm domain delete' - assert_text 'Domain registrant change has successfully received.' - - refute RegistrantVerification.find_by(verification_token:token).updator_str.empty? - end -end diff --git a/test/test_helper.rb b/test/test_helper.rb index 578c3ee9b..df2e9878e 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -18,6 +18,9 @@ require 'capybara/rails' require 'capybara/minitest' require 'webmock/minitest' require 'support/assertions/epp_assertions' +require 'sidekiq/testing' + +Sidekiq::Testing.fake! # `bin/rails test` is not the same as `bin/rake test`.