decrease db load

app/jobs/verify_emails_job.rb

@@ -1,12 +1,12 @@
 class VerifyEmailsJob < ApplicationJob
   discard_on StandardError
 
-  def perform(contact_id:, check_level: 'regex')
-    contact = Contact.find_by(id: contact_id)
+  def perform(contact:, check_level: 'regex')
+    # contact = Contact.find_by(id: contact_id)
 
-    return if check_contact_for_duplicate_mail(contact_id)
+    # return if check_contact_for_duplicate_mail(contact)
 
-    contact_not_found(contact_id) unless contact
+    contact_not_found(contact.id) unless contact
     validate_check_level(check_level)
     action = Actions::EmailCheck.new(email: contact.email,
                                      validation_eventable: contact,
@@ -19,15 +19,14 @@ class VerifyEmailsJob < ApplicationJob
 
   private
 
-  def check_contact_for_duplicate_mail(contact_id)
-    time = Time.zone.now - ValidationEvent::VALIDATION_PERIOD
-    contact = Contact.find(contact_id)
-    contact_ids = Contact.where(email: contact.email).where('created_at > ?', time).pluck(:id)
-
-    r = ValidationEvent.where(validation_eventable_id: contact_ids).order(created_at: :desc)
-
-    r.present?
-  end
+  # def check_contact_for_duplicate_mail(contact)
+  #   time = Time.zone.now - ValidationEvent::VALIDATION_PERIOD
+  #   contact_ids = Contact.where(email: contact.email).where('created_at > ?', time).pluck(:id)
+  #
+  #   r = ValidationEvent.where(validation_eventable_id: contact_ids).order(created_at: :desc)
+  #
+  #   r.present?
+  # end
 
   def contact_not_found(contact_id)
     raise StandardError, "Contact with contact_id #{contact_id} not found"

app/models/validation_event.rb

@@ -12,7 +12,7 @@ class ValidationEvent < ApplicationRecord
 
   INVALID_EVENTS_COUNT_BY_LEVEL = {
     regex: 1,
-    mx: 3,
+    mx: 2,
     smtp: 1,
   }.freeze
 

lib/tasks/generate_mock_contacts.rake

@@ -0,0 +1,28 @@
+# namespace :generate_mock do
+#   task contacts: :environment do
+#     1000.times do
+#       c = Contact.new
+#       c.name = generate_random_string
+#       c.email = generate_random_string + "@" + generate_random_string + ".ee"
+#       c.registrar_id = registrar
+#       c.street = generate_random_string
+#       c.city = generate_random_string
+#       c.zip = '12323'
+#       c.country_code = 'EE'
+#       c.phone = "+372.59813318"
+#       c.ident_country_code = 'EE'
+#       c.ident_type = 'priv'
+#       c.ident = '38903110313'
+#       c.code = generate_random_string + ":" + generate_random_string
+#       c.save
+#     end
+#   end
+#
+#   def generate_random_string
+#     (0...10).map { (65 + rand(26)).chr }.join
+#   end
+#
+#     def registrar
+#       Registrar.last.id
+#     end
+# end

lib/tasks/verify_email.rake

@@ -1,6 +1,7 @@
 require 'optparse'
 require 'rake_option_parser_boilerplate'
 require 'syslog/logger'
+require 'active_record'
 
 namespace :verify_email do
   # bundle exec rake verify_email:check_all -- --domain_name=shop.test --check_level=mx --spam_protect=true
@@ -18,14 +19,16 @@ namespace :verify_email do
                                                        banner: banner,
                                                        hash: opts_hash)
 
-    contacts = prepare_contacts(options)
-    logger.info 'No contacts to check email selected' and next if contacts.blank?
+    batch_contacts = prepare_contacts(options)
+    logger.info 'No contacts to check email selected' and next if batch_contacts.blank?
 
-    contacts.each do |contact|
-      VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)).perform_later(
-        contact_id: contact.id,
-        check_level: check_level(options)
-      )
+    batch_contacts.find_in_batches(batch_size: 10000) do |contacts|
+      contacts.each do |contact|
+        VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)).perform_later(
+          contact: contact,
+          check_level: check_level(options)
+        )
+      end
     end
   end
 end
@@ -55,29 +58,29 @@ def prepare_contacts(options)
     contacts_by_domain(options[:domain_name])
   else
     time = Time.zone.now - ValidationEvent::VALIDATION_PERIOD
-    validation_events_ids = ValidationEvent.where('created_at > ?', time).pluck(:validation_eventable_id)
+    validation_events_ids = ValidationEvent.where('created_at > ?', time).distinct.pluck(:validation_eventable_id)
 
-    # Contact.where.not(id: validation_events_ids) + Contact.where(id: failed_contacts)
-    Contact.where.not(id: validation_events_ids) | failed_contacts
+    contacts_ids = Contact.where.not(id: validation_events_ids).pluck(:id)
+    Contact.where(id: contacts_ids + failed_contacts)
   end
 end
 
 def failed_contacts
   failed_contacts = []
   failed_validations_ids = ValidationEvent.failed.pluck(:validation_eventable_id)
-  contacts = Contact.where(id: failed_validations_ids)
-  contacts.each do |contact|
+  contacts = Contact.where(id: failed_validations_ids).includes(:validation_events)
+  contacts.find_each(batch_size: 1000) do |contact|
 
     if contact.validation_events.mx.order(created_at: :asc).present?
-      failed_contacts << contact unless contact.validation_events.mx.order(created_at: :asc).last.success
+      failed_contacts << contact.id unless contact.validation_events.mx.order(created_at: :asc).last.success
     end
 
     if contact.validation_events.regex.order(created_at: :asc).present?
-      failed_contacts << contact unless contact.validation_events.regex.order(created_at: :asc).last.success
+      failed_contacts << contact.id unless contact.validation_events.regex.order(created_at: :asc).last.success
     end
 
     if contact.validation_events.smtp.order(created_at: :asc).present?
-      failed_contacts << contact unless contact.validation_events.mx.order(created_at: :asc).last.success
+      failed_contacts << contact.id unless contact.validation_events.mx.order(created_at: :asc).last.success
     end
   end