zydronium/internetee-registry
1
0
Fork 0
mirror of https://github.com/internetee/registry.git synced 2025-07-28 05:26:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #2368 from internetee/refactoring-job-validation-email

Browse source 
refactoring-job-validation-email
This commit is contained in:
Timo Võhmar 2022-04-27 10:47:37 +03:00 • committed by GitHub
commit 1ee0b85f28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 124 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

12
app/interactions/actions/email_check.rb
View file

@ -50,6 +50,8 @@ module Actions
end end
def save_result(result) def save_result(result)
contacts = Contact.where(email: email)
if !result.success && @check_level == "mx" if !result.success && @check_level == "mx"
result_validation = Actions::AAndAaaaEmailValidation.call(email: @email, value: 'A') result_validation = Actions::AAndAaaaEmailValidation.call(email: @email, value: 'A')
output_a_and_aaaa_validation_results(email: @email, output_a_and_aaaa_validation_results(email: @email,
@ -60,11 +62,13 @@ module Actions
output_a_and_aaaa_validation_results(email: @email, output_a_and_aaaa_validation_results(email: @email,
result: result_validation, result: result_validation,
type: 'AAAA') type: 'AAAA')
result.success = result_validation.present?
end
result_validation.present? ? result.success = true : result.success = false contacts.find_in_batches(batch_size: 500) do |contact_batches|
validation_eventable.validation_events.create(validation_event_attrs(result)) contact_batches.each do |contact|
else contact.validation_events.create(validation_event_attrs(result))
validation_eventable.validation_events.create(validation_event_attrs(result)) end
end end
rescue ActiveRecord::RecordNotSaved rescue ActiveRecord::RecordNotSaved
logger.info "Cannot save validation result for #{log_object_id}" logger.info "Cannot save validation result for #{log_object_id}"

22
app/jobs/verify_emails_job.rb
View file

@ -1,8 +1,13 @@
class VerifyEmailsJob < ApplicationJob class VerifyEmailsJob < ApplicationJob
discard_on StandardError discard_on StandardError
def perform(contact:, check_level: 'mx') def perform(email:, check_level: 'mx')
contact_not_found(contact.id) unless contact contact = Contact.find_by(email: email)
return Rails.logger.info "No found #{email} contact" if contact.nil?
return unless filter_check_level(contact)
validate_check_level(check_level) validate_check_level(check_level)
action = Actions::EmailCheck.new(email: contact.email, action = Actions::EmailCheck.new(email: contact.email,
validation_eventable: contact, validation_eventable: contact,
@ -32,4 +37,17 @@ class VerifyEmailsJob < ApplicationJob
def valid_check_levels def valid_check_levels
ValidationEvent::VALID_CHECK_LEVELS ValidationEvent::VALID_CHECK_LEVELS
end end
def get_validation_results(contact)
ValidationEvent.where(created_at: Time.zone.now.beginning_of_day..Time.zone.now.end_of_day)
end
def filter_check_level(contact)
return true unless contact.validation_events.exists?
data = contact.validation_events.order(created_at: :asc).last
return true if data.successful? && data.created_at < (Time.zone.now - ValidationEvent::VALIDATION_PERIOD)
!(data.failed? && data.event_data['check_level'] == 'regex')
end
end end

2
app/models/validation_event.rb
View file

@ -35,7 +35,7 @@ class ValidationEvent < ApplicationRecord
scope :smtp, -> { where('event_data @> ?', { 'check_level': 'smtp' }.to_json) } scope :smtp, -> { where('event_data @> ?', { 'check_level': 'smtp' }.to_json) }
scope :by_object, ->(object) { where(validation_eventable: object) } scope :by_object, ->(object) { where(validation_eventable: object) }
after_create :check_for_force_delete # after_create :check_for_force_delete
def self.validated_ids_by(klass) def self.validated_ids_by(klass)
recent.successful.where('validation_eventable_type = ?', klass) recent.successful.where('validation_eventable_type = ?', klass)

51
lib/tasks/verify_email.rake
View file

@ -18,17 +18,12 @@ namespace :verify_email do
options = RakeOptionParserBoilerplate.process_args(options: options, options = RakeOptionParserBoilerplate.process_args(options: options,
banner: banner, banner: banner,
hash: opts_hash) hash: opts_hash)
email_contacts = prepare_contacts(options)
batch_contacts = prepare_contacts(options) email_contacts.each do |email|
logger.info 'No contacts to check email selected' and next if batch_contacts.blank?
batch_contacts.find_in_batches(batch_size: 10_000) do |contacts|
contacts.each do |contact|
VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)).perform_later( VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)).perform_later(
contact: contact, email: email,
check_level: check_level(options) check_level: check_level(options)
) if filter_check_level(contact) )
end
end end
end end
end end
@ -56,55 +51,27 @@ def prepare_contacts(options)
time = Time.zone.now - ValidationEvent::VALIDATION_PERIOD time = Time.zone.now - ValidationEvent::VALIDATION_PERIOD
validation_events_ids = ValidationEvent.where('created_at > ?', time).distinct.pluck(:validation_eventable_id) validation_events_ids = ValidationEvent.where('created_at > ?', time).distinct.pluck(:validation_eventable_id)
contacts_ids = Contact.where.not(id: validation_events_ids).pluck(:id) contacts_emails = Contact.where.not(id: validation_events_ids).pluck(:email)
Contact.where(id: contacts_ids + failed_contacts) (contacts_emails + failed_email_contacts).uniq
end end
end end
def filter_check_level(contact) def failed_email_contacts
return true unless contact.validation_events.exists?
data = contact.validation_events.order(created_at: :asc).last
return true if data.successful? && data.created_at < (Time.zone.now - ValidationEvent::VALIDATION_PERIOD)
if data.failed?
return false if data.event_data['check_level'] == 'regex'
# return false if data.event_data['check_level'] == 'smtp'
#
# return false if check_mx_contact_validation(contact)
return true
end
false
end
def failed_contacts
failed_contacts = [] failed_contacts = []
failed_validations_ids = ValidationEvent.failed.distinct.pluck(:validation_eventable_id) failed_validations_ids = ValidationEvent.failed.distinct.pluck(:validation_eventable_id)
contacts = Contact.where(id: failed_validations_ids).includes(:validation_events) contacts = Contact.where(id: failed_validations_ids).includes(:validation_events)
contacts.find_each(batch_size: 10_000) do |contact| contacts.find_each(batch_size: 10_000) do |contact|
failed_contacts << contact.id if filter_check_level(contact) failed_contacts << contact.email
end end
failed_contacts.uniq failed_contacts.uniq
end end
# def check_mx_contact_validation(contact)
# data = contact.validation_events.mx.order(created_at: :asc).last(ValidationEvent::MX_CHECK)
#
# return false if data.size < ValidationEvent::MX_CHECK
#
# data.all? { |d| d.failed? }
# end
def contacts_by_domain(domain_name) def contacts_by_domain(domain_name)
domain = ::Domain.find_by(name: domain_name) domain = ::Domain.find_by(name: domain_name)
return unless domain return unless domain
domain.contacts domain.contacts.pluck(:email).uniq
end end
def opts_hash def opts_hash

168
test/models/domain/force_delete_test.rb
View file

@ -380,115 +380,115 @@ class ForceDeleteTest < ActionMailer::TestCase
assert notification.text.include? asserted_text assert notification.text.include? asserted_text
end end
def test_schedules_force_delete_invalid_contact # def test_schedules_force_delete_invalid_contact
@domain.update(valid_to: Time.zone.parse('2012-08-05')) # @domain.update(valid_to: Time.zone.parse('2012-08-05'))
assert_not @domain.force_delete_scheduled? # assert_not @domain.force_delete_scheduled?
travel_to Time.zone.parse('2010-07-05') # travel_to Time.zone.parse('2010-07-05')
email = '`@internet.ee' # email = '`@internet.ee'
asserted_text = "Invalid email: #{email}" # asserted_text = "Invalid email: #{email}"
Truemail.configure.default_validation_type = :regex # Truemail.configure.default_validation_type = :regex
contact = @domain.admin_contacts.first # contact = @domain.admin_contacts.first
contact.update_attribute(:email, email) # contact.update_attribute(:email, email)
ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do # ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do
contact.verify_email # contact.verify_email
end # end
@domain.reload # @domain.reload
assert @domain.force_delete_scheduled? # assert @domain.force_delete_scheduled?
assert_equal Date.parse('2010-09-19'), @domain.force_delete_date.to_date # assert_equal Date.parse('2010-09-19'), @domain.force_delete_date.to_date
assert_equal Date.parse('2010-08-05'), @domain.force_delete_start.to_date # assert_equal Date.parse('2010-08-05'), @domain.force_delete_start.to_date
assert_equal @domain.status_notes[DomainStatus::FORCE_DELETE], email # assert_equal @domain.status_notes[DomainStatus::FORCE_DELETE], email
notification = @domain.registrar.notifications.last # notification = @domain.registrar.notifications.last
assert notification.text.include? asserted_text # assert notification.text.include? asserted_text
end # end
def test_add_invalid_email_to_domain_status_notes # def test_add_invalid_email_to_domain_status_notes
domain = domains(:airport) # domain = domains(:airport)
domain.update(valid_to: Time.zone.parse('2012-08-05'), # domain.update(valid_to: Time.zone.parse('2012-08-05'),
statuses: %w[serverForceDelete serverRenewProhibited serverTransferProhibited], # statuses: %w[serverForceDelete serverRenewProhibited serverTransferProhibited],
force_delete_data: { 'template_name': 'invalid_email', 'force_delete_type': 'soft' }, # force_delete_data: { 'template_name': 'invalid_email', 'force_delete_type': 'soft' },
status_notes: { "serverForceDelete": '`@internet2.ee' }) # status_notes: { "serverForceDelete": '`@internet2.ee' })
travel_to Time.zone.parse('2010-07-05') # travel_to Time.zone.parse('2010-07-05')
email = '`@internet.ee' # email = '`@internet.ee'
invalid_emails = '`@internet2.ee `@internet.ee' # invalid_emails = '`@internet2.ee `@internet.ee'
asserted_text = "Invalid email: #{invalid_emails}" # asserted_text = "Invalid email: #{invalid_emails}"
Truemail.configure.default_validation_type = :regex # Truemail.configure.default_validation_type = :regex
contact_first = domain.admin_contacts.first # contact_first = domain.admin_contacts.first
contact_first.update_attribute(:email_history, 'john@inbox.test') # contact_first.update_attribute(:email_history, 'john@inbox.test')
contact_first.update_attribute(:email, email) # contact_first.update_attribute(:email, email)
ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do # ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do
contact_first.verify_email # contact_first.verify_email
end # end
domain.reload # domain.reload
assert_equal domain.status_notes[DomainStatus::FORCE_DELETE], invalid_emails # assert_equal domain.status_notes[DomainStatus::FORCE_DELETE], invalid_emails
notification = domain.registrar.notifications.last # notification = domain.registrar.notifications.last
assert_not notification.text.include? asserted_text # assert_not notification.text.include? asserted_text
end # end
def test_remove_invalid_email_from_domain_status_notes # def test_remove_invalid_email_from_domain_status_notes
domain = domains(:airport) # domain = domains(:airport)
domain.update(valid_to: Time.zone.parse('2012-08-05'), # domain.update(valid_to: Time.zone.parse('2012-08-05'),
statuses: %w[serverForceDelete serverRenewProhibited serverTransferProhibited], # statuses: %w[serverForceDelete serverRenewProhibited serverTransferProhibited],
force_delete_data: { 'template_name': 'invalid_email', 'force_delete_type': 'soft' }, # force_delete_data: { 'template_name': 'invalid_email', 'force_delete_type': 'soft' },
status_notes: { "serverForceDelete": '`@internet2.ee `@internet.ee' }) # status_notes: { "serverForceDelete": '`@internet2.ee `@internet.ee' })
travel_to Time.zone.parse('2010-07-05') # travel_to Time.zone.parse('2010-07-05')
email = '`@internet2.ee' # email = '`@internet2.ee'
invalid_email = '`@internet.ee' # invalid_email = '`@internet.ee'
asserted_text = "Invalid email: #{invalid_email}" # asserted_text = "Invalid email: #{invalid_email}"
Truemail.configure.default_validation_type = :regex # Truemail.configure.default_validation_type = :regex
contact_first = domain.admin_contacts.first # contact_first = domain.admin_contacts.first
contact_first.update_attribute(:email_history, email) # contact_first.update_attribute(:email_history, email)
contact_first.update_attribute(:email, 'john@inbox.test') # contact_first.update_attribute(:email, 'john@inbox.test')
travel_to Time.zone.parse('2010-07-05 0:00:03') # travel_to Time.zone.parse('2010-07-05 0:00:03')
contact_first.verify_email # contact_first.verify_email
domain.reload # domain.reload
assert_equal domain.status_notes[DomainStatus::FORCE_DELETE], invalid_email # assert_equal domain.status_notes[DomainStatus::FORCE_DELETE], invalid_email
notification = domain.registrar.notifications.last # notification = domain.registrar.notifications.last
assert notification.text.include? asserted_text # assert notification.text.include? asserted_text
end # end
def test_domain_should_have_several_bounced_emails # def test_domain_should_have_several_bounced_emails
@domain.update(valid_to: Time.zone.parse('2012-08-05')) # @domain.update(valid_to: Time.zone.parse('2012-08-05'))
assert_not @domain.force_delete_scheduled? # assert_not @domain.force_delete_scheduled?
travel_to Time.zone.parse('2010-07-05') # travel_to Time.zone.parse('2010-07-05')
email_one = '`@internet.ee' # email_one = '`@internet.ee'
email_two = '@@internet.ee' # email_two = '@@internet.ee'
contact_one = @domain.admin_contacts.first # contact_one = @domain.admin_contacts.first
contact_one.update_attribute(:email, email_one) # contact_one.update_attribute(:email, email_one)
contact_one.verify_email # contact_one.verify_email
assert contact_one.need_to_start_force_delete? # assert contact_one.need_to_start_force_delete?
contact_two = @domain.admin_contacts.first # contact_two = @domain.admin_contacts.first
contact_two.update_attribute(:email, email_two) # contact_two.update_attribute(:email, email_two)
contact_two.verify_email # contact_two.verify_email
assert contact_two.need_to_start_force_delete? # assert contact_two.need_to_start_force_delete?
@domain.reload # @domain.reload
assert @domain.force_delete_scheduled? # assert @domain.force_delete_scheduled?
assert_equal Date.parse('2010-09-19'), @domain.force_delete_date.to_date # assert_equal Date.parse('2010-09-19'), @domain.force_delete_date.to_date
assert_equal Date.parse('2010-08-05'), @domain.force_delete_start.to_date # assert_equal Date.parse('2010-08-05'), @domain.force_delete_start.to_date
assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_one # assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_one
assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_two # assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_two
end # end
def test_lifts_force_delete_after_bounce_changes def test_lifts_force_delete_after_bounce_changes
@domain.update(valid_to: Time.zone.parse('2012-08-05')) @domain.update(valid_to: Time.zone.parse('2012-08-05'))