Merge pull request #2063 from internetee/2059-email-verification-rework

Refactor email/contact verification

app/helpers/application_helper.rb

@@ -108,14 +108,4 @@ module ApplicationHelper
   def body_css_class
     [controller_path.split('/').map!(&:dasherize), action_name.dasherize, 'page'].join('-')
   end
-
-  def verified_email_span(verification)
-    tag.span(verification.email, class: verified_email_class(verification))
-  end
-
-  def verified_email_class(verification)
-    return 'text-danger' if verification.failed?
-    return 'text-primary' if verification.not_verified?
-    return 'text-success' if verification.verified?
-  end
 end

app/interactions/actions/email_check.rb

@@ -0,0 +1,64 @@
+module Actions
+  class EmailCheck
+    attr_reader :email, :validation_eventable, :check_level
+
+    def initialize(email:, validation_eventable:, check_level: nil)
+      @email = email
+      @validation_eventable = validation_eventable
+      @check_level = check_level || :regex
+    end
+
+    def call
+      result = check_email(email)
+      save_result(result)
+      result.success ? log_success : log_failure(result)
+      result.success
+    end
+
+    private
+
+    def check_email(parsed_email)
+      Truemail.validate(parsed_email, with: calculate_check_level).result
+    end
+
+    def calculate_check_level
+      Rails.env.test? && check_level == 'smtp' ? :mx : check_level.to_sym
+    end
+
+    def save_result(result)
+      validation_eventable.validation_events.create(validation_event_attrs(result))
+    rescue ActiveRecord::RecordNotSaved
+      logger.info "Cannot save validation result for #{log_object_id}"
+      true
+    end
+
+    def validation_event_attrs(result)
+      {
+        event_data: event_data(result),
+        event_type: ValidationEvent::EventType::TYPES[:email_validation],
+        success: result.success,
+      }
+    end
+
+    def logger
+      @logger ||= Rails.logger
+    end
+
+    def event_data(result)
+      result.to_h.merge(check_level: check_level)
+    end
+
+    def log_failure(result)
+      logger.info "Failed to validate email #{email} for the #{log_object_id}."
+      logger.info "Validation level #{check_level}, the result was #{result}"
+    end
+
+    def log_success
+      logger.info "Successfully validated email #{email} for the #{log_object_id}."
+    end
+
+    def log_object_id
+      "#{validation_eventable.class}: #{validation_eventable.id}"
+    end
+  end
+end

app/interactions/domains/force_delete_lift/base.rb

@@ -6,8 +6,6 @@ module Domains
              description: 'Domain to check if ForceDelete needs to be listed'
 
       def execute
-        prepare_email_verifications(domain)
-
         lift_force_delete(domain) if force_delete_condition(domain)
       end
 
@@ -29,13 +27,8 @@ module Domains
       end
 
       def contact_emails_valid?(domain)
-        domain.contacts.all? { |contact| contact.email_verification.verified? } &&
+        domain.contacts.all(&:need_to_lift_force_delete?) &&
-          domain.registrant.email_verification.verified?
+          domain.registrant.need_to_lift_force_delete?
-      end
-
-      def prepare_email_verifications(domain)
-        domain.registrant.email_verification.verify
-        domain.contacts.each { |contact| contact.email_verification.verify }
       end
 
       def bounces_absent?(domain)

app/jobs/verify_emails_job.rb

@@ -1,47 +1,37 @@
 class VerifyEmailsJob < ApplicationJob
   discard_on StandardError
 
-  def perform(verification_id)
+  def perform(contact_id:, check_level: 'regex')
-    email_address_verification = EmailAddressVerification.find(verification_id)
+    contact = Contact.find_by(id: contact_id)
-    return unless need_to_verify?(email_address_verification)
+    contact_not_found(contact_id) unless contact
+    validate_check_level(check_level)
 
-    process(email_address_verification)
+    action = Actions::EmailCheck.new(email: contact.email,
+                                     validation_eventable: contact,
+                                     check_level: check_level)
+    action.call
   rescue StandardError => e
-    log_error(verification: email_address_verification, error: e)
+    logger.error e.message
     raise e
   end
 
   private
 
-  def need_to_verify?(email_address_verification)
+  def contact_not_found(contact_id)
-    return false if email_address_verification.blank?
+    raise StandardError, "Contact with contact_id #{contact_id} not found"
-    return false if email_address_verification.recently_verified?
-
-    true
   end
 
-  def process(email_address_verification)
+  def validate_check_level(check_level)
-    email_address_verification.verify
+    return if valid_check_levels.include? check_level
-    log_success(email_address_verification)
+
+    raise StandardError, "Check level #{check_level} is invalid"
   end
 
   def logger
-    @logger ||= Logger.new(Rails.root.join('log/email_verification.log'))
+    @logger ||= Rails.logger
   end
 
-  def log_success(verification)
+  def valid_check_levels
-    email = verification.try(:email) || verification
+    ValidationEvent::VALID_CHECK_LEVELS
-    message = "Email address #{email} verification done"
-    logger.info message
-  end
-
-  def log_error(verification:, error:)
-    email = verification.try(:email) || verification
-    message = <<~TEXT.squish
-      There was an error verifying email #{email}.
-      The error message was the following: #{error}
-      This job will retry.
-    TEXT
-    logger.error message
   end
 end

app/models/concerns/email_verifable.rb

@@ -1,91 +1,59 @@
 module EmailVerifable
   extend ActiveSupport::Concern
 
-  def email_verification
+  included do
-    EmailAddressVerification.find_or_create_by(email: unicode_email, domain: domain(email))
+    scope :recently_not_validated, -> { where.not(id: ValidationEvent.validated_ids_by(name)) }
-  end
-
-  def billing_email_verification
-    return unless attribute_names.include?('billing_email')
-
-    EmailAddressVerification.find_or_create_by(email: unicode_billing_email,
-                                               domain: domain(billing_email))
   end
 
   def email_verification_failed?
-    email_verification&.failed?
+    need_to_start_force_delete?
   end
 
-  class_methods do
+  def need_to_start_force_delete?
-    def domain(email)
+    ValidationEvent::INVALID_EVENTS_COUNT_BY_LEVEL.any? do |level, count|
-      Mail::Address.new(email).domain&.downcase || 'not_found'
+      validation_events.recent.order(id: :desc).limit(count).all? do |event|
-    rescue Mail::Field::IncompleteParseError
+        event.check_level == level.to_s && event.failed?
-      'not_found'
       end
-
-    def local(email)
-      Mail::Address.new(email).local&.downcase || email
-    rescue Mail::Field::IncompleteParseError
-      email
-    end
-
-    def punycode_to_unicode(email)
-      return email if domain(email) == 'not_found'
-
-      local = local(email)
-      domain = SimpleIDN.to_unicode(domain(email))
-      "#{local}@#{domain}"&.downcase
-    end
-
-    def unicode_to_punycode(email)
-      return email if domain(email) == 'not_found'
-
-      local = local(email)
-      domain = SimpleIDN.to_ascii(domain(email))
-      "#{local}@#{domain}"&.downcase
     end
   end
 
-  def unicode_billing_email
+  def need_to_lift_force_delete?
-    self.class.punycode_to_unicode(billing_email)
+    validation_events.recent.failed.empty? ||
+      ValidationEvent::REDEEM_EVENTS_COUNT_BY_LEVEL.any? do |level, count|
+        validation_events.recent.order(id: :desc).limit(count).all? do |event|
+          event.check_level == level.to_s && event.successful?
         end
-
-  def unicode_email
-    self.class.punycode_to_unicode(email)
       end
-
-  def domain(email)
-    SimpleIDN.to_unicode(self.class.domain(email))
-  end
-
-  def punycode_to_unicode(email)
-    self.class.punycode_to_unicode(email)
   end
 
   def correct_email_format
     return if email.blank?
 
-    result = email_verification.verify
+    result = verify(email: email)
-    process_result(result: result, field: :email)
+    process_error(:email) unless result
   end
 
   def correct_billing_email_format
     return if email.blank?
 
-    result = billing_email_verification.verify
+    result = verify(email: billing_email)
-    process_result(result: result, field: :billing_email)
+    process_error(:billing_email) unless result
+  end
+
+  def verify_email(check_level: 'regex')
+    verify(email: email, check_level: check_level)
+  end
+
+  def verify(email:, check_level: 'regex')
+    action = Actions::EmailCheck.new(email: email,
+                                     validation_eventable: self,
+                                     check_level: check_level)
+    action.call
   end
 
   # rubocop:disable Metrics/LineLength
-  def process_result(result:, field:)
+  def process_error(field)
-    case result[:errors].keys.first
-    when :smtp
-      errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'))
-    when :mx
-      errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_mx_check_error'))
-    when :regex
     errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error'))
   end
-  end
   # rubocop:enable Metrics/LineLength
 end

app/models/contact.rb

@@ -16,6 +16,7 @@ class Contact < ApplicationRecord
   has_many :domain_contacts
   has_many :domains, through: :domain_contacts
   has_many :legal_documents, as: :documentable
+  has_many :validation_events, as: :validation_eventable
   has_many :registrant_domains, class_name: 'Domain', foreign_key: 'registrant_id'
   has_many :actions, dependent: :destroy
 

app/models/email_address_verification.rb

@@ -1,67 +0,0 @@
-class EmailAddressVerification < ApplicationRecord
-  RECENTLY_VERIFIED_PERIOD = 1.month
-  after_save :check_force_delete
-
-  scope :not_verified_recently, lambda {
-    where('verified_at IS NULL or verified_at < ?', verification_period)
-  }
-
-  scope :verified_recently, lambda {
-    where('verified_at IS NOT NULL and verified_at >= ?', verification_period).where(success: true)
-  }
-
-  scope :verification_failed, lambda {
-    where.not(verified_at: nil).where(success: false)
-  }
-
-  scope :by_domain, ->(domain_name) { where(domain: domain_name) }
-
-  def recently_verified?
-    verified_at.present? &&
-      verified_at > verification_period
-  end
-
-  def verification_period
-    self.class.verification_period
-  end
-
-  def self.verification_period
-    Time.zone.now - RECENTLY_VERIFIED_PERIOD
-  end
-
-  def not_verified?
-    verified_at.blank? && !success
-  end
-
-  def failed?
-    bounce_present? || (verified_at.present? && !success)
-  end
-
-  def verified?
-    success
-  end
-
-  def bounce_present?
-    BouncedMailAddress.find_by(email: email).present?
-  end
-
-  def check_force_delete
-    return unless failed?
-
-    Domains::ForceDeleteEmail::Base.run(email: email)
-  end
-
-  def verify
-    validation_request = Truemail.validate(email)
-
-    if validation_request.result.success
-      update(verified_at: Time.zone.now,
-             success: true)
-    else
-      update(verified_at: Time.zone.now,
-             success: false)
-    end
-
-    validation_request.result
-  end
-end

app/models/registrar.rb

@@ -13,6 +13,7 @@ class Registrar < ApplicationRecord
   has_many :nameservers, through: :domains
   has_many :whois_records
   has_many :white_ips, dependent: :destroy
+  has_many :validation_events, as: :validation_eventable
 
   delegate :balance, to: :cash_account, allow_nil: true
 

app/models/validation_event.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# Class to store validation events. Need to include boolean `success` field - was validation event
+# successful or not.
+# Types of events supported so far stored in ValidationEvent::EventType::TYPES
+# For email_validation event kind also check_level (regex/mx/smtp) is stored in the event_data
+class ValidationEvent < ApplicationRecord
+  enum event_type: ValidationEvent::EventType::TYPES, _suffix: true
+  VALIDATION_PERIOD = 1.month.freeze
+  VALID_CHECK_LEVELS = %w[regex mx smtp].freeze
+  VALID_EVENTS_COUNT_THRESHOLD = 5
+
+  INVALID_EVENTS_COUNT_BY_LEVEL = {
+    regex: 1,
+    mx: 5,
+    smtp: 1,
+  }.freeze
+
+  REDEEM_EVENTS_COUNT_BY_LEVEL = {
+    regex: 1,
+    mx: 1,
+    smtp: 1,
+  }.freeze
+
+  store_accessor :event_data, :errors, :check_level, :email
+
+  belongs_to :validation_eventable, polymorphic: true
+
+  scope :recent, -> { where('created_at > ?', Time.zone.now - VALIDATION_PERIOD) }
+  scope :successful, -> { where(success: true) }
+  scope :failed, -> { where(success: false) }
+  scope :regex, -> { where('event_data @> ?', { 'check_level': 'regex' }.to_json) }
+  scope :mx, -> { where('event_data @> ?', { 'check_level': 'mx' }.to_json) }
+  scope :smtp, -> { where('event_data @> ?', { 'check_level': 'smtp' }.to_json) }
+  scope :by_object, ->(object) { where(validation_eventable: object) }
+
+  after_create :check_for_force_delete
+
+  def self.validated_ids_by(klass)
+    recent.successful.where('validation_eventable_type = ?', klass)
+          .pluck(:validation_eventable_id)
+  end
+
+  def failed?
+    !success
+  end
+
+  def successful?
+    success
+  end
+
+  def event_type
+    @event_type ||= ValidationEvent::EventType.new(self[:event_type])
+  end
+
+  def object
+    validation_eventable
+  end
+
+  def check_for_force_delete
+    if object.need_to_start_force_delete?
+      start_force_delete
+    elsif object.need_to_lift_force_delete?
+      lift_force_delete
+    end
+  end
+
+  def start_force_delete
+    Domains::ForceDeleteEmail::Base.run(email: email)
+  end
+
+  def lift_force_delete; end
+end

app/models/validation_event/event_type.rb

@@ -0,0 +1,10 @@
+class ValidationEvent
+  class EventType
+    TYPES = { email_validation: 'email_validation',
+              manual_force_delete: 'manual_force_delete' }.freeze
+
+    def initialize(event_type)
+      @event_type = event_type
+    end
+  end
+end

app/views/admin/contacts/index.haml

@@ -101,7 +101,7 @@
               %td= link_to(contact, admin_contact_path(contact))
               %td= contact.code
               %td= ident_for(contact)
-              %td= verified_email_span(contact.email_verification)
+              %td= contact.email
               %td= l(contact.created_at, format: :short)
               %td
                 - if contact.registrar

app/views/admin/contacts/partials/_general.haml

@@ -17,7 +17,7 @@
       %dd.left_25= ident_for(@contact)
 
       %dt.left_25= t(:email)
-      %dd.left_25= verified_email_span(@contact.email_verification)
+      %dd.left_25= @contact.email
 
       %dt.left_25= t(:phone)
       %dd.left_25= @contact.phone

app/views/admin/registrars/index.html.erb

@@ -53,9 +53,9 @@
                             <%= "#{x.test_registrar}" %>
                         </td>
                         <td>
-                            <%= verified_email_span(x.email_verification) %>
+                            <%= content_tag(:span, x.email) %>
                             <% if x[:billing_email].present? %>
-                                <%= verified_email_span(x.billing_email_verification) %>
+                                <%= content_tag(:span, x[:billing_email]) %>
                             <% end %>
                         </td>
                     </tr>

app/views/admin/registrars/show/_billing.html.erb

@@ -16,7 +16,7 @@
 
             <dt><%= Registrar.human_attribute_name :billing_email %></dt>
             <dd>
-                <%= verified_email_span(registrar.billing_email_verification) %>
+                <%= registrar.billing_email %>
             </dd>
 
             <dt><%= Registrar.human_attribute_name :reference_no %></dt>

app/views/admin/registrars/show/_contacts.html.erb

@@ -16,7 +16,7 @@
 
             <dt><%= Registrar.human_attribute_name :email %></dt>
             <dd>
-                <%= verified_email_span(@registrar.email_verification) %>
+                <%= @registrar.email %>
             </dd>
         </dl>
     </div>

db/migrate/20210629074044_create_validation_events.rb

@@ -0,0 +1,28 @@
+class CreateValidationEvents < ActiveRecord::Migration[6.1]
+
+  def up
+    execute <<-SQL
+      CREATE TYPE validation_type AS ENUM ('email_validation', 'manual_force_delete');
+    SQL
+
+    create_table :validation_events do |t|
+      t.jsonb :event_data
+      t.boolean :success
+      t.references :validation_eventable, polymorphic: true
+
+      t.timestamps
+    end
+
+    add_column :validation_events, :event_type, :validation_type
+    add_index :validation_events, :event_type
+  end
+
+  def down
+    remove_column :validation_events, :event_type
+    execute <<-SQL
+      DROP TYPE validation_type;
+    SQL
+
+    drop_table :validation_events
+  end
+end

db/structure.sql

@@ -65,6 +65,16 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
 COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions';
 
 
+--
+-- Name: validation_type; Type: TYPE; Schema: public; Owner: -
+--
+
+CREATE TYPE public.validation_type AS ENUM (
+    'email_validation',
+    'manual_force_delete'
+);
+
+
 --
 -- Name: generate_zonefile(character varying); Type: FUNCTION; Schema: public; Owner: -
 --
@@ -2595,6 +2605,41 @@ CREATE SEQUENCE public.users_id_seq
 ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id;
 
 
+--
+-- Name: validation_events; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.validation_events (
+    id bigint NOT NULL,
+    event_data jsonb,
+    success boolean,
+    validation_eventable_type character varying,
+    validation_eventable_id bigint,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL,
+    event_type public.validation_type
+);
+
+
+--
+-- Name: validation_events_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.validation_events_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: validation_events_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.validation_events_id_seq OWNED BY public.validation_events.id;
+
+
 --
 -- Name: versions; Type: TABLE; Schema: public; Owner: -
 --
@@ -3170,6 +3215,13 @@ ALTER TABLE ONLY public.settings ALTER COLUMN id SET DEFAULT nextval('public.set
 ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass);
 
 
+--
+-- Name: validation_events id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.validation_events ALTER COLUMN id SET DEFAULT nextval('public.validation_events_id_seq'::regclass);
+
+
 --
 -- Name: versions id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -3814,6 +3866,14 @@ ALTER TABLE ONLY public.users
     ADD CONSTRAINT users_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: validation_events validation_events_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.validation_events
+    ADD CONSTRAINT validation_events_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: versions versions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -3986,13 +4046,6 @@ CREATE INDEX index_domain_transfers_on_domain_id ON public.domain_transfers USIN
 CREATE INDEX index_domains_on_delete_date ON public.domains USING btree (delete_date);
 
 
---
--- Name: index_domains_on_json_statuses_history; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_domains_on_json_statuses_history ON public.domains USING gin (json_statuses_history);
-
-
 --
 -- Name: index_domains_on_name; Type: INDEX; Schema: public; Owner: -
 --
@@ -4434,6 +4487,20 @@ CREATE INDEX index_users_on_identity_code ON public.users USING btree (identity_
 CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_id);
 
 
+--
+-- Name: index_validation_events_on_event_type; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_validation_events_on_event_type ON public.validation_events USING btree (event_type);
+
+
+--
+-- Name: index_validation_events_on_validation_eventable; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_validation_events_on_validation_eventable ON public.validation_events USING btree (validation_eventable_type, validation_eventable_id);
+
+
 --
 -- Name: index_versions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -5161,6 +5228,8 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20200921084356'),
 ('20210215101019'),
 ('20210616112332'),
+('20210629074044'),
+('20210628090353'),
 ('20210708131814');
 
 

lib/rake_option_parser_boilerplate.rb

@@ -0,0 +1,14 @@
+module RakeOptionParserBoilerplate
+  module_function
+
+  def process_args(options:, banner:, hash: {})
+    o = OptionParser.new
+    o.banner = banner
+    hash.each do |command_line_argument, setup_value|
+      o.on(*setup_value) { |result| options[command_line_argument] = result }
+    end
+    args = o.order!(ARGV) {}
+    o.parse!(args)
+    options
+  end
+end

lib/tasks/verify_email.rake

@@ -1,23 +1,70 @@
+require 'optparse'
+require 'rake_option_parser_boilerplate'
+require 'syslog/logger'
+
 namespace :verify_email do
-  desc 'Stars verifying email jobs for all the domain'
+  # bundle exec rake verify_email:check_all -- --domain_name=shop.test --check_level=mx --spam_protect=true
-  task all_domains: :environment do
+  # bundle exec rake verify_email:check_all -- -dshop.test -cmx -strue
-    verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain)
+  desc 'Starts verifying email jobs with optional check level and spam protection'
-    verifications_by_domain.each do |_domain, verifications|
+  task check_all: :environment do
-      ver = verifications.sample # Verify random email to not to clog the SMTP servers
+    SPAM_PROTECT_TIMEOUT = 30.seconds
-      VerifyEmailsJob.perform_later(ver.id)
+    options = {
-      next
+      domain_name: nil,
-    end
+      check_level: 'regex',
-  end
+      spam_protect: false,
+    }
+    banner = 'Usage: rake verify_email:check_all -- [options]'
+    options = RakeOptionParserBoilerplate.process_args(options: options,
+                                                       banner: banner,
+                                                       hash: opts_hash)
 
-  # Need to be run like 'bundle exec rake verify_email:domain['gmail.com']'
+    contacts = prepare_contacts(options)
-  # In zsh syntax will be 'bundle exec rake verify_email:domain\['gmail.com'\]'
+    logger.info 'No contacts to check email selected' and next if contacts.blank?
-  # Default 'bundle exec rake verify_email:domain' wil use 'internet.ee' domain
-  desc 'Stars verifying email jobs for domain stated in argument'
-  task :domain, [:domain_name] => [:environment] do |_task, args|
-    args.with_defaults(domain_name: 'internet.ee')
 
-    verifications_by_domain = EmailAddressVerification.not_verified_recently
+    contacts.find_each do |contact|
-                                                      .by_domain(args[:domain_name])
+      VerifyEmailsJob.set(wait_until: spam_protect_timeout(options)).perform_later(
-    verifications_by_domain.map { |ver| VerifyEmailsJob.perform_later(ver.id) }
+        contact_id: contact.id,
+        check_level: check_level(options)
+      )
+    end
   end
 end
+
+def check_level(options)
+  options[:check_level]
+end
+
+def spam_protect(options)
+  options[:spam_protect]
+end
+
+def spam_protect_timeout(options)
+  spam_protect(options) ? 0.seconds : SPAM_PROTECT_TIMEOUT
+end
+
+def logger
+  @logger ||= ActiveSupport::TaggedLogging.new(Syslog::Logger.new('registry'))
+end
+
+def prepare_contacts(options)
+  if options[:domain_name].present?
+    contacts_by_domain(options[:domain_name])
+  else
+    Contact.recently_not_validated
+  end
+end
+
+def contacts_by_domain(domain_name)
+  domain = ::Domain.find_by(name: domain_name)
+  return unless domain
+
+  domain.contacts.recently_not_validated
+end
+
+def opts_hash
+  {
+    domain_name: ['-d [DOMAIN_NAME]', '--domain_name [DOMAIN_NAME]', String],
+    check_level: ['-c [CHECK_LEVEL]', '--check_level [CHECK_LEVEL]', String],
+    spam_protect: ['-s [SPAM_PROTECT]', '--spam_protect [SPAM_PROTECT]', FalseClass],
+  }
+end

test/integration/epp/domain/update/base_test.rb

@@ -814,7 +814,7 @@ class EppDomainUpdateBaseTest < EppTestCase
   def test_makes_update_if_was_forcedelete
     contact = @domain.contacts.first
     contact.update_attribute(:email, '`@outlook.test')
-    contact.email_verification.verify
+    contact.verify_email
     assert contact.email_verification_failed?
     @domain.reload
     assert @domain.force_delete_scheduled?

test/jobs/verify_emails_job_test.rb

@@ -4,9 +4,6 @@ class VerifyEmailsJobTest < ActiveJob::TestCase
   def setup
     @contact = contacts(:john)
     @invalid_contact = contacts(:invalid_email)
-    @contact_verification = @contact.email_verification
-    @invalid_contact_verification = @invalid_contact.email_verification
-
     @default_whitelist = Truemail.configure.whitelisted_domains
     @default_blacklist = Truemail.configure.blacklisted_domains
     Truemail.configure.whitelisted_domains = whitelisted_domains
@@ -33,33 +30,21 @@ class VerifyEmailsJobTest < ActiveJob::TestCase
   end
 
   def test_job_checks_if_email_valid
+    assert_difference 'ValidationEvent.successful.count', 1 do
       perform_enqueued_jobs do
-      VerifyEmailsJob.perform_now(@contact_verification.id)
+        VerifyEmailsJob.perform_now(contact_id: @contact.id, check_level: 'regex')
       end
-    @contact_verification.reload
-
-    assert @contact_verification.success
     end
-
+    assert ValidationEvent.validated_ids_by(Contact).include? @contact.id
-  def test_job_checks_does_not_run_if_recent
-    old_verified_at = Time.zone.now - 10.days
-    @contact_verification.update(success: true, verified_at: old_verified_at)
-    assert @contact_verification.recently_verified?
-
-    perform_enqueued_jobs do
-      VerifyEmailsJob.perform_now(@contact_verification.id)
-    end
-    @contact_verification.reload
-
-    assert_in_delta @contact_verification.verified_at.to_i, old_verified_at.to_i, 1
   end
 
   def test_job_checks_if_email_invalid
     perform_enqueued_jobs do
-      VerifyEmailsJob.perform_now(@invalid_contact_verification.id)
+      VerifyEmailsJob.perform_now(contact_id: @invalid_contact.id, check_level: 'regex')
     end
-    @contact_verification.reload
+    @invalid_contact.reload
 
-    refute @contact_verification.success
+    refute @invalid_contact.validation_events.last.success
+    refute ValidationEvent.validated_ids_by(Contact).include? @invalid_contact.id
   end
 end

test/mailers/domain_expire_mailer_test.rb

@@ -40,7 +40,7 @@ class DomainExpireMailerTest < ActionMailer::TestCase
 
     contact = domain.admin_contacts.first
     contact.update_attribute(:email, email)
-    contact.email_verification.verify
+    contact.verify_email
 
     assert contact.email_verification_failed?
 

test/models/bounced_mail_address_test.rb

@@ -138,7 +138,7 @@ class BouncedMailAddressTest < ActiveSupport::TestCase
     registrant = domains(:shop).registrant
 
     assert_equal registrant.email, bounced_mail.email
-    assert registrant.email_verification.failed?
+    assert registrant.email_verification_failed?
   end
 
   def sns_bounce_payload

test/models/contact_test.rb

@@ -87,24 +87,6 @@ class ContactTest < ActiveJob::TestCase
     assert contact.valid?
   end
 
-  def test_email_verification_smtp_error
-    Truemail.configure.default_validation_type = :smtp
-
-    contact = valid_contact
-    contact.email = 'somecrude1337joke@internet.ee'
-    assert contact.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), contact.errors.messages[:email].first
- end
-
-  def test_email_verification_mx_error
-    Truemail.configure.default_validation_type = :mx
-
-    contact = valid_contact
-    contact.email = 'somecrude31337joke@somestrange31337domain.ee'
-    assert contact.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_mx_check_error'), contact.errors.messages[:email].first
-  end
-
   def test_email_verification_regex_error
     Truemail.configure.default_validation_type = :regex
 
@@ -358,16 +340,6 @@ class ContactTest < ActiveJob::TestCase
     assert_equal domain.whois_record.try(:json).try(:[], 'registrant'), @contact.name
   end
 
-  def test_creates_email_verification_in_unicode
-    unicode_email = 'suur@äri.ee'
-    punycode_email = Contact.unicode_to_punycode(unicode_email)
-
-    @contact.email = punycode_email
-    @contact.save
-
-    assert_equal @contact.email_verification.email, unicode_email
-  end
-
   private
 
   def make_contact_free_of_domains_where_it_acts_as_a_registrant(contact)

test/models/domain/force_delete_test.rb

@@ -6,6 +6,7 @@ class ForceDeleteTest < ActionMailer::TestCase
     Setting.redemption_grace_period = 30
     ActionMailer::Base.deliveries.clear
     @old_validation_type = Truemail.configure.default_validation_type
+    ValidationEvent.destroy_all
   end
 
   teardown do
@@ -393,7 +394,10 @@ class ForceDeleteTest < ActionMailer::TestCase
 
     contact = @domain.admin_contacts.first
     contact.update_attribute(:email, email)
-    contact.email_verification.verify
+
+    ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do
+      contact.verify_email
+    end
 
     assert contact.email_verification_failed?
 
@@ -414,20 +418,18 @@ class ForceDeleteTest < ActionMailer::TestCase
     travel_to Time.zone.parse('2010-07-05')
     email_one = '`@internet.ee'
     email_two = '@@internet.ee'
-    asserted_text_one = "Invalid email: #{email_one}"
-    asserted_text_two = "Invalid email: #{email_two}"
 
     contact_one = @domain.admin_contacts.first
     contact_one.update_attribute(:email, email_one)
-    contact_one.email_verification.verify
+    contact_one.verify_email
 
-    assert contact_one.email_verification_failed?
+    assert contact_one.need_to_start_force_delete?
 
     contact_two = @domain.admin_contacts.first
     contact_two.update_attribute(:email, email_two)
-    contact_two.email_verification.verify
+    contact_two.verify_email
 
-    assert contact_one.email_verification_failed?
+    assert contact_two.need_to_start_force_delete?
 
     @domain.reload
 
@@ -440,16 +442,16 @@ class ForceDeleteTest < ActionMailer::TestCase
   end
 
   def test_lifts_force_delete_if_contact_fixed
+    travel_to Time.zone.parse('2010-07-05')
     @domain.update(valid_to: Time.zone.parse('2012-08-05'))
     assert_not @domain.force_delete_scheduled?
-    travel_to Time.zone.parse('2010-07-05')
     email = '`@internet.ee'
 
     Truemail.configure.default_validation_type = :regex
 
     contact = @domain.admin_contacts.first
     contact.update_attribute(:email, email)
-    contact.email_verification.verify
+    contact.verify_email
 
     assert contact.email_verification_failed?
 
@@ -457,7 +459,11 @@ class ForceDeleteTest < ActionMailer::TestCase
 
     assert @domain.force_delete_scheduled?
     contact.update_attribute(:email, 'aaa@bbb.com')
-    contact.email_verification.verify
+    contact.reload
+    contact.verify_email
+
+    assert contact.need_to_lift_force_delete?
+    refute contact.need_to_start_force_delete?
 
     assert_not contact.email_verification_failed?
     CheckForceDeleteLift.perform_now
@@ -486,8 +492,8 @@ class ForceDeleteTest < ActionMailer::TestCase
     assert notification.text.include? asserted_text
 
     @domain.registrant.update(email: 'aaa@bbb.com')
-    @domain.registrant.email_verification.verify
+    @domain.registrant.verify_email
-    assert_not @domain.registrant.email_verification_failed?
+    assert @domain.registrant.need_to_lift_force_delete?
     CheckForceDeleteLift.perform_now
 
     @domain.reload

test/models/registrar_test.rb

@@ -48,28 +48,6 @@ class RegistrarTest < ActiveJob::TestCase
     assert registrar.valid?
   end
 
-  def test_email_verification_smtp_error
-    Truemail.configure.default_validation_type = :smtp
-
-    registrar = valid_registrar
-    registrar.email = 'somecrude1337joke@internet.ee'
-    registrar.billing_email = nil
-
-    assert registrar.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), registrar.errors.messages[:email].first
-  end
-
-  def test_email_verification_mx_error
-    Truemail.configure.default_validation_type = :mx
-
-    registrar = valid_registrar
-    registrar.email = 'somecrude31337joke@somestrange31337domain.ee'
-    registrar.billing_email = nil
-
-    assert registrar.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_mx_check_error'), registrar.errors.messages[:email].first
-  end
-
   def test_email_verification_regex_error
     Truemail.configure.default_validation_type = :regex
 
@@ -88,26 +66,6 @@ class RegistrarTest < ActiveJob::TestCase
     assert registrar.valid?
   end
 
-  def test_billing_email_verification_smtp_error
-    Truemail.configure.default_validation_type = :smtp
-
-    registrar = valid_registrar
-    registrar.billing_email = 'somecrude1337joke@internet.ee'
-
-    assert registrar.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), registrar.errors.messages[:billing_email].first
-  end
-
-  def test_billing_email_verification_mx_error
-    Truemail.configure.default_validation_type = :mx
-
-    registrar = valid_registrar
-    registrar.billing_email = 'somecrude31337joke@somestrange31337domain.ee'
-
-    assert registrar.invalid?
-    assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_mx_check_error'), registrar.errors.messages[:billing_email].first
-  end
-
   def test_billing_email_verification_regex_error
     Truemail.configure.default_validation_type = :regex
 
@@ -118,21 +76,6 @@ class RegistrarTest < ActiveJob::TestCase
     assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error'), registrar.errors.messages[:billing_email].first
   end
 
-  def test_creates_email_verification_in_unicode
-    unicode_email = 'suur@äri.ee'
-    punycode_email = Registrar.unicode_to_punycode(unicode_email)
-    unicode_billing_email = 'billing@äri.ee'
-    punycode_billing_email = Registrar.unicode_to_punycode(unicode_billing_email)
-
-    registrar = valid_registrar
-    registrar.email = punycode_email
-    registrar.billing_email = punycode_billing_email
-    registrar.save
-
-    assert_equal registrar.email_verification.email, unicode_email
-    assert_equal registrar.billing_email_verification.email, unicode_billing_email
-  end
-
   def test_invalid_without_accounting_customer_code
     registrar = valid_registrar
     registrar.accounting_customer_code = ''

test/models/validation_event_test.rb

@@ -0,0 +1,107 @@
+require 'test_helper'
+
+class ValidationEventTest < ActiveSupport::TestCase
+
+  setup do
+    @domain = domains(:shop)
+    Setting.redemption_grace_period = 30
+    ActionMailer::Base.deliveries.clear
+  end
+
+  teardown do
+
+  end
+
+  def test_if_fd_need_to_be_set_if_invalid_email
+    @domain.update(valid_to: Time.zone.parse('2012-08-05'))
+    assert_not @domain.force_delete_scheduled?
+    travel_to Time.zone.parse('2010-07-05')
+    email = '~@internet.ee'
+
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+    contact.verify_email
+    contact.reload
+
+    refute contact.validation_events.last.success?
+    assert contact.need_to_start_force_delete?
+  end
+
+  def test_if_fd_need_to_be_lifted_if_email_fixed
+    test_if_fd_need_to_be_set_if_invalid_email
+
+    email = 'email@internet.ee'
+
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+
+    contact.verify_email
+    contact.reload
+
+    assert contact.need_to_lift_force_delete?
+    assert contact.validation_events.last.success?
+  end
+
+  def test_if_fd_need_to_be_set_if_invalid_mx
+    @domain.update(valid_to: Time.zone.parse('2012-08-05'))
+    assert_not @domain.force_delete_scheduled?
+    travel_to Time.zone.parse('2010-07-05')
+
+    email = 'email@somestrangedomain12345.ee'
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+    ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do
+      contact.verify_email(check_level: 'mx')
+    end
+    contact.reload
+
+    refute contact.validation_events.limit(ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD)
+                  .any?(&:success?)
+    assert contact.need_to_start_force_delete?
+  end
+
+  def test_if_fd_need_to_be_lifted_if_mx_fixed
+    test_if_fd_need_to_be_set_if_invalid_mx
+
+    email = 'email@internet.ee'
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+    contact.verify_email(check_level: 'mx')
+
+    contact.reload
+    assert contact.need_to_lift_force_delete?
+    assert contact.validation_events.last.success?
+  end
+
+  def test_if_fd_need_to_be_set_if_invalid_smtp
+    @domain.update(valid_to: Time.zone.parse('2012-08-05'))
+    assert_not @domain.force_delete_scheduled?
+    travel_to Time.zone.parse('2010-07-05')
+
+    email = 'email@somestrangedomain12345.ee'
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+    ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD.times do
+      contact.verify_email(check_level: 'smtp')
+    end
+    contact.reload
+
+    refute contact.validation_events.limit(ValidationEvent::VALID_EVENTS_COUNT_THRESHOLD)
+                  .any?(&:success?)
+    assert contact.need_to_start_force_delete?
+  end
+
+  def test_if_fd_need_to_be_lifted_if_smtp_fixed
+    test_if_fd_need_to_be_set_if_invalid_smtp
+
+    email = 'valid@internet.ee'
+    contact = @domain.admin_contacts.first
+    contact.update_attribute(:email, email)
+    contact.verify_email(check_level: 'smtp')
+
+    contact.reload
+    assert contact.need_to_lift_force_delete?
+    assert contact.validation_events.last.success?
+  end
+
+end

test/system/admin_area/domains/force_delete_test.rb

@@ -61,8 +61,12 @@ class AdminAreaDomainForceDeleteTest < ApplicationSystemTestCase
   end
 
   def test_uses_legal_template_if_invalid_email
-    verification = @domain.contacts.first.email_verification
+    contact = @domain.contacts.first
-    verification.update(verified_at: Time.zone.now - 1.day, success: false)
+    contact.update(email: '`@domain.com`')
+    action = Actions::EmailCheck.new(email: contact.email, validation_eventable: contact)
+    action.call
+
+    @domain.reload
 
     assert_equal @domain.notification_template, 'invalid_email'
 

test/tasks/emails/verify_email_task_test.rb

@@ -5,8 +5,6 @@ class VerifyEmailTaskTest < ActiveJob::TestCase
   def setup
     @contact = contacts(:john)
     @invalid_contact = contacts(:invalid_email)
-    @contact_verification = @contact.email_verification
-    @invalid_contact_verification = @invalid_contact.email_verification
 
     @default_whitelist = Truemail.configure.whitelisted_domains
     @default_blacklist = Truemail.configure.blacklisted_domains
@@ -36,32 +34,15 @@ class VerifyEmailTaskTest < ActiveJob::TestCase
   def test_tasks_verifies_emails
     capture_io { run_task }
 
-    @contact_verification.reload
+    assert ValidationEvent.validated_ids_by(Contact).include? @contact.id
-    @invalid_contact_verification.reload
+    assert @contact.validation_events.last.success
-
+    refute @invalid_contact.validation_events.last.success
-    assert @contact_verification.verified?
+    refute ValidationEvent.validated_ids_by(Contact).include? @invalid_contact.id
-    assert @invalid_contact_verification.failed?
-  end
-
-  def test_domain_task_verifies_for_one_domain
-    capture_io { run_single_domain_task(@contact_verification.domain) }
-
-    @contact_verification.reload
-    @invalid_contact_verification.reload
-
-    assert @contact_verification.verified?
-    assert @invalid_contact_verification.not_verified?
   end
 
   def run_task
     perform_enqueued_jobs do
-      Rake::Task['verify_email:all_domains'].execute
+      Rake::Task['verify_email:check_all'].execute
-    end
-  end
-
-  def run_single_domain_task(domain)
-    perform_enqueued_jobs do
-      Rake::Task["verify_email:domain"].invoke(domain)
     end
   end
 end