From e4a02c2e47e1dbd1c113ec79664d820b1ba4a5b5 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 13 Jul 2020 15:04:56 +0500 Subject: [PATCH] Return truemail This reverts commit 862f5639ebbe4d3e6abd5d5be7fb7840e7b83bdf. --- .codeclimate.yml | 4 + .travis.yml | 3 +- Gemfile | 6 +- Gemfile.lock | 94 +++--- app/controllers/admin/contacts_controller.rb | 14 +- app/helpers/application_helper.rb | 10 + app/jobs/verify_emails_job.rb | 45 +++ app/models/concerns/email_verifable.rb | 2 + app/models/contact.rb | 9 +- app/models/email_address_verification.rb | 56 ++++ app/models/epp/contact.rb | 4 +- app/models/nameserver.rb | 2 +- app/models/registrar.rb | 8 +- app/views/admin/contacts/index.haml | 9 +- .../admin/contacts/partials/_general.haml | 2 +- app/views/admin/registrars/index.html.erb | 9 + .../admin/registrars/show/_billing.html.erb | 6 +- .../admin/registrars/show/_contacts.html.erb | 6 +- config/initializers/airbrake.rb | 2 +- config/initializers/truemail.rb | 6 +- config/locales/admin/email_verifable.en.yml | 5 + config/locales/admin/email_verifable.et.yml | 5 + config/locales/en.yml | 1 + config/locales/et.yml | 1 + config/schedule.rb | 4 + ...20200608084321_fill_email_verifications.rb | 21 ++ ...0827_create_email_address_verifications.rb | 13 + ...nge_email_verification_fields_to_citext.rb | 13 + db/structure.sql | 304 +++++++++++++----- lib/tasks/verify_email.rake | 23 ++ test/fixtures/contacts.yml | 8 + test/jobs/verify_emails_job_test.rb | 59 ++++ test/models/contact_test.rb | 26 +- test/models/registrar_test.rb | 38 ++- test/tasks/emails/verify_email_task_test.rb | 63 ++++ 35 files changed, 728 insertions(+), 153 deletions(-) create mode 100644 app/jobs/verify_emails_job.rb create mode 100644 app/models/email_address_verification.rb create mode 100644 config/locales/admin/email_verifable.en.yml create mode 100644 config/locales/admin/email_verifable.et.yml create mode 100644 db/data/20200608084321_fill_email_verifications.rb create mode 100644 db/migrate/20200605100827_create_email_address_verifications.rb create mode 100644 db/migrate/20200610090110_change_email_verification_fields_to_citext.rb create mode 100644 lib/tasks/verify_email.rake create mode 100644 test/jobs/verify_emails_job_test.rb create mode 100644 test/tasks/emails/verify_email_task_test.rb diff --git a/.codeclimate.yml b/.codeclimate.yml index 2d22653b8..2bc90b200 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -23,6 +23,10 @@ plugins: rubocop: enabled: true channel: rubocop-0-74 +checks: + method-lines: + config: + threshold: 40 exclude_patterns: - "app/models/version/" - "bin/" diff --git a/.travis.yml b/.travis.yml index 01373f29d..bb74deecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: ruby cache: bundler env: - DB=postgresql -bundler_args: --without development staging production before_install: - "wget -N http://chromedriver.storage.googleapis.com/2.43/chromedriver_linux64.zip -P ~/" - "unzip ~/chromedriver_linux64.zip -d ~/" @@ -10,6 +9,8 @@ before_install: - "sudo mv -f ~/chromedriver /usr/local/share/" - "sudo chmod +x /usr/local/share/chromedriver" - "sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver" + - "bundle config set without 'development staging production'" + - "bundle config set deployment '[secure]'" before_script: - "cp config/application.yml.sample config/application.yml" - "echo \"openssl_config_path: 'test/fixtures/files/test_ca/openssl.cnf'\" >> config/application.yml" diff --git a/Gemfile b/Gemfile index f6c8a6397..99091f556 100644 --- a/Gemfile +++ b/Gemfile @@ -11,10 +11,12 @@ gem 'uglifier' gem 'figaro', '1.1.1' # model related +gem 'activerecord-import' gem 'paper_trail', '~> 10.3' gem 'pg', '1.2.2' # 1.8 is for Rails < 5.0 gem 'ransack', '~> 2.3' +gem 'truemail', '~> 1.7' # validates email by regexp, mail server existence and address existence gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and RFC 3696 # 0.7.3 is the latest for Rails 4.2, however, it is absent on Rubygems server @@ -39,7 +41,7 @@ gem 'grape' # registry specfic gem 'data_migrate', '~> 6.1' gem 'isikukood' # for EE-id validation -gem 'simpleidn', '0.0.9' # For punycode +gem 'simpleidn', '0.1.1' # For punycode gem 'money-rails' gem 'whenever', '0.9.4', require: false @@ -67,7 +69,7 @@ gem 'e_invoice', github: 'internetee/e_invoice', branch: :master gem 'lhv', github: 'internetee/lhv', branch: :master gem 'domain_name' gem 'haml', '~> 5.0' -gem 'wkhtmltopdf-binary' +gem 'wkhtmltopdf-binary', '~> 0.12.5.1' gem 'directo', github: 'internetee/directo', branch: 'master' diff --git a/Gemfile.lock b/Gemfile.lock index effb7ff38..e4ad26396 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ GIT GIT remote: https://github.com/internetee/directo.git - revision: bdfab4be20803c666dcefc9a9c607f915a056ac5 + revision: 8ff8a382d004ffb85722a6a7a68a020bd4d7159b branch: master specs: directo (1.0.1) @@ -112,6 +112,8 @@ GEM activerecord (6.0.3.2) activemodel (= 6.0.3.2) activesupport (= 6.0.3.2) + activerecord-import (1.0.5) + activerecord (>= 3.2) activestorage (6.0.3.2) actionpack (= 6.0.3.2) activejob (= 6.0.3.2) @@ -125,18 +127,18 @@ GEM zeitwerk (~> 2.2, >= 2.2.2) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) - airbrake (10.0.1) + airbrake (10.0.5) airbrake-ruby (~> 4.13) - airbrake-ruby (4.13.0) + airbrake-ruby (4.15.0) rbtree3 (~> 0.5) akami (1.3.1) gyoku (>= 0.4.0) nokogiri - autodoc (0.7.3) + autodoc (0.7.4) actionpack activesupport (>= 3.0.0) rspec - autoprefixer-rails (9.7.4) + autoprefixer-rails (9.8.4) execjs bcrypt (3.1.13) bootsnap (1.4.6) @@ -145,8 +147,8 @@ GEM autoprefixer-rails (>= 5.2.1) sassc (>= 2.0.0) builder (3.2.4) - cancancan (3.0.2) - capybara (3.31.0) + cancancan (3.1.0) + capybara (3.33.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -165,8 +167,8 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.1.6) - countries (3.0.0) - i18n_data (~> 0.8.0) + countries (3.0.1) + i18n_data (~> 0.10.0) sixarm_ruby_unaccent (~> 1.1) unicode_utils (~> 1.4) crack (0.4.3) @@ -176,22 +178,23 @@ GEM daemons-rails (1.2.1) daemons multi_json (~> 1.0) - data_migrate (6.2.0) + data_migrate (6.3.0) rails (>= 5.0) - database_cleaner (1.8.2) - devise (4.7.1) + database_cleaner (1.8.5) + devise (4.7.2) bcrypt (~> 3.0) orm_adapter (~> 0.1) railties (>= 4.1.0) responders warden (~> 1.2.3) - diff-lcs (1.3) + diff-lcs (1.4.4) docile (1.3.2) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - dry-configurable (0.9.0) + dry-configurable (0.11.6) concurrent-ruby (~> 1.0) dry-core (~> 0.4, >= 0.4.7) + dry-equalizer (~> 0.2) dry-container (0.7.2) concurrent-ruby (~> 1.0) dry-configurable (~> 0.1, >= 0.1.3) @@ -199,11 +202,11 @@ GEM concurrent-ruby (~> 1.0) dry-equalizer (0.3.0) dry-inflector (0.2.0) - dry-logic (1.0.5) + dry-logic (1.0.6) concurrent-ruby (~> 1.0) dry-core (~> 0.2) dry-equalizer (~> 0.2) - dry-types (1.2.2) + dry-types (1.4.0) concurrent-ruby (~> 1.0) dry-container (~> 0.3) dry-core (~> 0.4, >= 0.4.4) @@ -213,12 +216,12 @@ GEM erubi (1.9.0) erubis (2.7.0) execjs (2.7.0) - ffi (1.12.2) + ffi (1.13.1) figaro (1.1.1) thor (~> 0.14) globalid (0.4.2) activesupport (>= 4.2.0) - grape (1.3.0) + grape (1.3.3) activesupport builder dry-types (>= 1.1) @@ -230,7 +233,7 @@ GEM haml (5.1.2) temple (>= 0.8.0) tilt - hashdiff (1.0.0) + hashdiff (1.0.1) hpricot (0.8.6) http-accept (1.7.0) http-cookie (1.0.3) @@ -241,10 +244,10 @@ GEM socksify i18n (1.8.3) concurrent-ruby (~> 1.0) - i18n_data (0.8.0) + i18n_data (0.10.0) isikukood (0.1.2) iso8601 (0.12.1) - jquery-rails (4.3.5) + jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) @@ -264,7 +267,7 @@ GEM kaminari-core (= 1.2.1) kaminari-core (1.2.1) keystores (0.4.0) - libxml-ruby (3.1.0) + libxml-ruby (3.2.0) listen (3.2.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -279,7 +282,7 @@ GEM method_source (0.8.2) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2019.1009) + mime-types-data (3.2020.0512) mimemagic (0.3.5) mina (0.3.1) open4 (~> 1.3.4) @@ -289,7 +292,7 @@ GEM minitest (5.14.1) monetize (1.9.4) money (~> 6.12) - money (6.13.7) + money (6.13.8) i18n (>= 0.6.4, <= 2) money-rails (1.13.3) activesupport (>= 3.0) @@ -304,7 +307,7 @@ GEM mustermann (>= 1.0.0) netrc (0.11.0) nio4r (2.5.2) - nokogiri (1.10.9) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) nori (2.6.0) open4 (1.3.4) @@ -312,7 +315,7 @@ GEM paper_trail (10.3.1) activerecord (>= 4.2) request_store (~> 1.1) - pdfkit (0.8.4.1) + pdfkit (0.8.4.3.1) pg (1.2.2) polyamorous (2.3.2) activerecord (>= 5.2.1) @@ -320,7 +323,7 @@ GEM coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - public_suffix (4.0.3) + public_suffix (4.0.5) puma (4.3.5) nio4r (~> 2.0) que (0.14.3) @@ -375,10 +378,10 @@ GEM ffi (~> 1.0) rbtree3 (0.6.0) rdoc (4.3.0) - regexp_parser (1.6.0) + regexp_parser (1.7.1) request_store (1.5.0) rack (>= 1.4) - responders (3.0.0) + responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) rest-client (2.1.0) @@ -390,21 +393,21 @@ GEM rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) rspec-mocks (~> 3.9.0) - rspec-core (3.9.1) - rspec-support (~> 3.9.1) - rspec-expectations (3.9.0) + rspec-core (3.9.2) + rspec-support (~> 3.9.3) + rspec-expectations (3.9.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) rspec-mocks (3.9.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) - rspec-support (3.9.2) + rspec-support (3.9.3) ruby2_keywords (0.0.2) - rubyzip (2.2.0) + rubyzip (2.3.0) safe_yaml (1.0.5) sass-rails (6.0.0) sassc-rails (~> 2.1, >= 2.1.1) - sassc (2.2.1) + sassc (2.4.0) ffi (~> 1.9) sassc-rails (2.1.2) railties (>= 4.0.0) @@ -412,7 +415,7 @@ GEM sprockets (> 3.0) sprockets-rails tilt - savon (2.12.0) + savon (2.12.1) akami (~> 1.2) builder (>= 2.1.2) gyoku (~> 1.2) @@ -434,7 +437,8 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - simpleidn (0.0.9) + simpleidn (0.1.1) + unf (~> 0.1.4) sinatra (2.0.8.1) mustermann (~> 1.0) rack (~> 2.0) @@ -454,13 +458,15 @@ GEM thor (0.20.3) thread_safe (0.3.6) tilt (2.0.10) + truemail (1.8.0) + simpleidn (~> 0.1.1) tzinfo (1.2.7) thread_safe (~> 0.1) uglifier (4.2.0) execjs (>= 0.3.0, < 3) unf (0.1.4) unf_ext - unf_ext (0.0.7.6) + unf_ext (0.0.7.7) unicode_utils (1.4.0) validates_email_format_of (1.6.3) i18n @@ -469,11 +475,11 @@ GEM wasabi (3.5.0) httpi (~> 2.0) nokogiri (>= 1.4.2) - webdrivers (4.2.0) + webdrivers (4.4.1) nokogiri (~> 1.6) rubyzip (>= 1.3.0) selenium-webdriver (>= 3.0, < 4.0) - webmock (3.8.0) + webmock (3.8.3) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) @@ -482,7 +488,7 @@ GEM websocket-extensions (0.1.5) whenever (0.9.4) chronic (>= 0.6.3) - wkhtmltopdf-binary (0.12.5.1) + wkhtmltopdf-binary (0.12.5.4) xpath (3.2.0) nokogiri (~> 1.8) zeitwerk (2.3.1) @@ -491,6 +497,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import airbrake autodoc bootsnap (>= 1.1.0) @@ -542,13 +549,14 @@ DEPENDENCIES select2-rails (= 3.5.9.3) selectize-rails (= 0.12.1) simplecov (= 0.17.1) - simpleidn (= 0.0.9) + simpleidn (= 0.1.1) + truemail (~> 1.7) uglifier validates_email_format_of (= 1.6.3) webdrivers webmock whenever (= 0.9.4) - wkhtmltopdf-binary + wkhtmltopdf-binary (~> 0.12.5.1) BUNDLED WITH 2.1.4 diff --git a/app/controllers/admin/contacts_controller.rb b/app/controllers/admin/contacts_controller.rb index 4eea4faad..793fa1209 100644 --- a/app/controllers/admin/contacts_controller.rb +++ b/app/controllers/admin/contacts_controller.rb @@ -13,10 +13,10 @@ module Admin search_params[:registrant_domains_id_not_null] = 1 end - contacts = Contact.includes(:registrar).joins(:registrar).select('contacts.*, registrars.name') + contacts = Contact.includes(:registrar).joins(:registrar) + .select('contacts.*, registrars.name') contacts = contacts.filter_by_states(params[:statuses_contains].join(',')) if params[:statuses_contains] - contacts = contacts.where("ident_country_code is null or ident_country_code=''") if params[:only_no_country_code].eql?('1') - + contacts = filter_by_flags(contacts) normalize_search_parameters do @q = contacts.search(search_params) @@ -26,6 +26,14 @@ module Admin @contacts = @contacts.per(params[:results_per_page]) if params[:results_per_page].to_i.positive? end + def filter_by_flags(contacts) + if params[:only_no_country_code].eql?('1') + contacts = contacts.where("ident_country_code is null or ident_country_code=''") + end + contacts = contacts.email_verification_failed if params[:email_verification_failed].eql?('1') + contacts + end + def search render json: Contact.search_by_query(params[:q]) end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 8de3fdc70..5c742afce 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -108,4 +108,14 @@ module ApplicationHelper def body_css_class [controller_path.split('/').map!(&:dasherize), action_name.dasherize, 'page'].join('-') end + + def verified_email_span(verification) + content_tag(:span, verification.email, class: verified_email_class(verification)) + end + + def verified_email_class(verification) + return 'text-danger' if verification.failed? + return 'text-primary' if verification.not_verified? + return 'text-success' if verification.verified? + end end diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb new file mode 100644 index 000000000..75f4b7d91 --- /dev/null +++ b/app/jobs/verify_emails_job.rb @@ -0,0 +1,45 @@ +class VerifyEmailsJob < Que::Job + def run(verification_id) + email_address_verification = run_condition(EmailAddressVerification.find(verification_id)) + + return if email_address_verification.recently_verified? + + ActiveRecord::Base.transaction do + email_address_verification.verify + log_success(email_address_verification) + destroy + end + rescue StandardError => e + log_error(verification: email_address_verification, error: e) + raise e + end + + private + + def run_condition(email_address_verification) + destroy unless email_address_verification + destroy if email_address_verification.recently_verified? + + email_address_verification + end + + def logger + @logger ||= Logger.new(Rails.root.join('log', 'email_verification.log')) + end + + def log_success(verification) + email = verification.try(:email) || verification + message = "Email address #{email} verification done" + logger.info message + end + + def log_error(verification:, error:) + email = verification.try(:email) || verification + message = <<~TEXT.squish + There was an error verifying email #{email}. + The error message was the following: #{error} + This job will retry. + TEXT + logger.error message + end +end diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index f573b5e44..dc512b2c8 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -75,6 +75,7 @@ module Concerns process_result(result: result, field: :billing_email) end + # rubocop:disable Metrics/LineLength def process_result(result:, field:) case result[:errors].keys.first when :smtp @@ -85,5 +86,6 @@ module Concerns errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error')) end end + # rubocop:enable Metrics/LineLength end end diff --git a/app/models/contact.rb b/app/models/contact.rb index f07c0c114..ac64b059f 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -7,6 +7,7 @@ class Contact < ApplicationRecord include Concerns::Contact::Transferable include Concerns::Contact::Identical include Concerns::Contact::Disclosable + include Concerns::EmailVerifable belongs_to :original, class_name: self.name belongs_to :registrar, required: true @@ -22,6 +23,11 @@ class Contact < ApplicationRecord accepts_nested_attributes_for :legal_documents + scope :email_verification_failed, lambda { + joins('LEFT JOIN email_address_verifications emv ON contacts.email = emv.email') + .where('success = false and verified_at IS NOT NULL') + } + validates :name, :email, presence: true validates :street, :city, :zip, :country_code, presence: true, if: lambda { self.class.address_processing? @@ -29,8 +35,7 @@ class Contact < ApplicationRecord validates :phone, presence: true, e164: true, phone: true - validates :email, format: /@/ - validates :email, email_format: { message: :invalid }, if: proc { |c| c.will_save_change_to_email? } + validate :correct_email_format, if: proc { |c| c.will_save_change_to_email? } validates :code, uniqueness: { message: :epp_id_taken }, diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb new file mode 100644 index 000000000..2fe7c0dbe --- /dev/null +++ b/app/models/email_address_verification.rb @@ -0,0 +1,56 @@ +class EmailAddressVerification < ApplicationRecord + RECENTLY_VERIFIED_PERIOD = 1.month + + scope :not_verified_recently, lambda { + where('verified_at IS NULL or verified_at < ?', verification_period) + } + + scope :verified_recently, lambda { + where('verified_at IS NOT NULL and verified_at >= ?', verification_period).where(success: true) + } + + scope :verification_failed, lambda { + where.not(verified_at: nil).where(success: false) + } + + scope :by_domain, ->(domain_name) { where(domain: domain_name) } + + def recently_verified? + verified_at.present? && + verified_at > verification_period + end + + def verification_period + self.class.verification_period + end + + def self.verification_period + Time.zone.now - RECENTLY_VERIFIED_PERIOD + end + + def not_verified? + verified_at.blank? && !success + end + + def failed? + verified_at.present? && !success + end + + def verified? + success + end + + def verify + validation_request = Truemail.validate(email) + + if validation_request.result.success + update(verified_at: Time.zone.now, + success: true) + else + update(verified_at: Time.zone.now, + success: false) + end + + validation_request.result + end +end diff --git a/app/models/epp/contact.rb b/app/models/epp/contact.rb index 82dd57aa4..6867b037d 100644 --- a/app/models/epp/contact.rb +++ b/app/models/epp/contact.rb @@ -60,6 +60,7 @@ class Epp::Contact < Contact delegate :ident_attr_valid?, to: :class + # rubocop:disable Style/SymbolArray def epp_code_map { '2003' => [ # Required parameter missing @@ -80,7 +81,7 @@ class Epp::Contact < Contact [:code, :too_long_contact_code], [:email, :email_smtp_check_error], [:email, :email_mx_check_error], - [:email, :email_regex_check_error] + [:email, :email_regex_check_error], ], '2302' => [ # Object exists [:code, :epp_id_taken] @@ -90,6 +91,7 @@ class Epp::Contact < Contact ] } end + # rubocop:enable Style/SymbolArray def attach_legal_document(legal_document_data) return unless legal_document_data diff --git a/app/models/nameserver.rb b/app/models/nameserver.rb index 3ddf1d1c5..3e4051165 100644 --- a/app/models/nameserver.rb +++ b/app/models/nameserver.rb @@ -88,7 +88,7 @@ class Nameserver < ApplicationRecord end def normalize_attributes - self.hostname = hostname.try(:strip).try(:downcase) + self.hostname = hostname.try(:strip).try(:downcase).gsub(/\.$/, '') self.ipv4 = Array(ipv4).reject(&:blank?).map(&:strip) self.ipv6 = Array(ipv6).reject(&:blank?).map(&:strip).map(&:upcase) end diff --git a/app/models/registrar.rb b/app/models/registrar.rb index dbdd7e8d3..470d768b7 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -1,6 +1,7 @@ class Registrar < ApplicationRecord include Versions # version/registrar_version.rb include Concerns::Registrar::BookKeeping + include Concerns::EmailVerifable include Concerns::Registrar::LegalDoc has_many :domains, dependent: :restrict_with_error @@ -29,14 +30,11 @@ class Registrar < ApplicationRecord validates :vat_rate, numericality: { greater_than_or_equal_to: 0, less_than: 100 }, allow_nil: true - validate :forbid_special_code - attribute :vat_rate, ::Type::VATRate.new after_initialize :set_defaults - validates :email, email_format: { message: :invalid }, - allow_blank: true, if: proc { |c| c.will_save_change_to_email? } - validates :billing_email, email_format: { message: :invalid }, allow_blank: true + validate :correct_email_format, if: proc { |c| c.will_save_change_to_email? } + validate :correct_billing_email_format alias_attribute :contact_email, :email diff --git a/app/views/admin/contacts/index.haml b/app/views/admin/contacts/index.haml index cc80ac744..cbd11d3fc 100644 --- a/app/views/admin/contacts/index.haml +++ b/app/views/admin/contacts/index.haml @@ -63,6 +63,10 @@ .form-group = label_tag :only_no_country_code, "Ident CC missing" = check_box_tag :only_no_country_code, '1',params[:only_no_country_code].eql?('1'), style: 'width:auto;height:auto;float:right' + .col-md-3 + .form-group + = label_tag :email_verification_failed, "Email verification failed" + = check_box_tag :email_verification_failed, '1',params[:email_verification_failed].eql?('1'), style: 'width:auto;height:auto;float:right' .row .col-md-3{style: 'padding-top: 25px;float:right;'} @@ -85,7 +89,9 @@ %th{class: 'col-xs-2'} = sort_link(@q, 'ident', t(:ident)) %th{class: 'col-xs-2'} - = sort_link(@q, 'email', t(:created_at)) + = sort_link(@q, 'email', t(:email)) + %th{class: 'col-xs-2'} + = sort_link(@q, 'created_at', t(:created_at)) %th{class: 'col-xs-2'} = sort_link(@q, 'registrar_name', t(:registrar_name)) %tbody @@ -94,6 +100,7 @@ %td= link_to(contact, admin_contact_path(contact)) %td= contact.code %td= ident_for(contact) + %td= verified_email_span(contact.email_verification) %td= l(contact.created_at, format: :short) %td - if contact.registrar diff --git a/app/views/admin/contacts/partials/_general.haml b/app/views/admin/contacts/partials/_general.haml index 029f89509..6568cd3d0 100644 --- a/app/views/admin/contacts/partials/_general.haml +++ b/app/views/admin/contacts/partials/_general.haml @@ -17,7 +17,7 @@ %dd= ident_for(@contact) %dt= t(:email) - %dd= @contact.email + %dd= verified_email_span(@contact.email_verification) %dt= t(:phone) %dd= @contact.phone diff --git a/app/views/admin/registrars/index.html.erb b/app/views/admin/registrars/index.html.erb index a66816568..e641f5294 100644 --- a/app/views/admin/registrars/index.html.erb +++ b/app/views/admin/registrars/index.html.erb @@ -28,6 +28,9 @@ <%= t(:test_registrar) %> + + <%= t(:emails) %> + @@ -45,6 +48,12 @@ <%= "#{x.test_registrar}" %> + + <%= verified_email_span(x.email_verification) %> + <% if x[:billing_email].present? %> + <%= verified_email_span(x.billing_email_verification) %> + <% end %> + <% end %> diff --git a/app/views/admin/registrars/show/_billing.html.erb b/app/views/admin/registrars/show/_billing.html.erb index da79b9074..07bccc7f4 100644 --- a/app/views/admin/registrars/show/_billing.html.erb +++ b/app/views/admin/registrars/show/_billing.html.erb @@ -15,7 +15,9 @@
<%= registrar.accounting_customer_code %>
<%= Registrar.human_attribute_name :billing_email %>
-
<%= registrar.billing_email %>
+
+ <%= verified_email_span(registrar.billing_email_verification) %> +
<%= Registrar.human_attribute_name :reference_no %>
<%= registrar.reference_no %>
@@ -24,4 +26,4 @@
<%= registrar.iban %>
- \ No newline at end of file + diff --git a/app/views/admin/registrars/show/_contacts.html.erb b/app/views/admin/registrars/show/_contacts.html.erb index f467e6a51..0ca1158d3 100644 --- a/app/views/admin/registrars/show/_contacts.html.erb +++ b/app/views/admin/registrars/show/_contacts.html.erb @@ -15,7 +15,9 @@
<%= @registrar.phone %>
<%= Registrar.human_attribute_name :email %>
-
<%= @registrar.email %>
+
+ <%= verified_email_span(@registrar.email_verification) %> +
- \ No newline at end of file + diff --git a/config/initializers/airbrake.rb b/config/initializers/airbrake.rb index 5c1983369..abfe408c1 100644 --- a/config/initializers/airbrake.rb +++ b/config/initializers/airbrake.rb @@ -17,5 +17,5 @@ Airbrake.configure do |config| end config.environment = ENV['airbrake_env'] || Rails.env config.ignore_environments = %w[test] - config.blacklist_keys = Rails.application.config.filter_parameters + config.blocklist_keys = Rails.application.config.filter_parameters end diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb index 26e4e0dc5..88f244e8f 100644 --- a/config/initializers/truemail.rb +++ b/config/initializers/truemail.rb @@ -29,11 +29,9 @@ Truemail.configure do |config| # Truemail.validate('email@email.com') call without with-parameter # Available validation types: :regex, :mx, :smtp if Rails.env.production? - config.default_validation_type = :smtp - elsif Rails.env.test? - config.default_validation_type = :regex - else config.default_validation_type = :mx + else + config.default_validation_type = :regex end # Optional parameter. You can predefine which type of validation will be used for domains. diff --git a/config/locales/admin/email_verifable.en.yml b/config/locales/admin/email_verifable.en.yml new file mode 100644 index 000000000..724fa4c32 --- /dev/null +++ b/config/locales/admin/email_verifable.en.yml @@ -0,0 +1,5 @@ +en: + email_verifable: + email_smtp_check_error: SMTP check error + email_mx_check_error: Mail domain not found + email_regex_check_error: Invalid format diff --git a/config/locales/admin/email_verifable.et.yml b/config/locales/admin/email_verifable.et.yml new file mode 100644 index 000000000..6c008ed11 --- /dev/null +++ b/config/locales/admin/email_verifable.et.yml @@ -0,0 +1,5 @@ +et: + email_verifable: + email_smtp_check_error: Eposti aadressi ei leitud (SMTP viga) + email_mx_check_error: Eposti aadressi domeeni ei leitud + email_regex_check_error: Eposti aadress on vigane diff --git a/config/locales/en.yml b/config/locales/en.yml index a825b1dc0..27299072e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -636,6 +636,7 @@ en: edit_dispute: 'Edit dispute' optional: 'Optional' test_registrar: "Test registrar" + emails: 'Email addresses' verified_confirm: 'Verified status is for cases when current registrant is the one applying for the update. Legal document signed by the registrant is required. Are you sure this update is properly verified with the registrant?' verified: 'Verified' deleted: 'Deleted' diff --git a/config/locales/et.yml b/config/locales/et.yml index 05d32be24..9cb8aaa4a 100644 --- a/config/locales/et.yml +++ b/config/locales/et.yml @@ -5,3 +5,4 @@ et: date: # Don't forget the nil at the beginning; there's no such thing as a 0th month month_names: [~, Jaanuar, Veebruar, Märts, Aprill, Mai, Juuni, Juuli, August, September, Oktoober, November, Detsember] + emails: "Meillaadressid" diff --git a/config/schedule.rb b/config/schedule.rb index 089ce93f9..7ebf97d12 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -62,6 +62,10 @@ if @cron_group == 'registry' rake 'domain:discard' end + every 10.minutes do + rake 'verify_email:all_domains' + end + # Should be at least once every 4 days, since according to LHV specs: # "Unread messages older than 5 days are automatically scheduled for deletion" # https://partners.lhv.ee/en/connect/#messaging diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb new file mode 100644 index 000000000..37a7f275c --- /dev/null +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -0,0 +1,21 @@ +class FillEmailVerifications < ActiveRecord::Migration[6.0] + include Concerns::EmailVerifable + + def up + registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?) + registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?) + contact_emails = Contact.pluck(:email).uniq.reject(&:blank?) + + emails = (contact_emails + registrar_emails + registrar_billing_emails) + emails = emails.map{ |email| punycode_to_unicode(email) }.uniq + + result = emails.map do |email| + { email: email, domain: domain(email) } + end + EmailAddressVerification.import result, batch_size: 500 + end + + def down + EmailAddressVerification.delete_all + end +end diff --git a/db/migrate/20200605100827_create_email_address_verifications.rb b/db/migrate/20200605100827_create_email_address_verifications.rb new file mode 100644 index 000000000..7f618b3a7 --- /dev/null +++ b/db/migrate/20200605100827_create_email_address_verifications.rb @@ -0,0 +1,13 @@ +class CreateEmailAddressVerifications < ActiveRecord::Migration[6.0] + def change + create_table :email_address_verifications do |t| + t.string :email, null: false + t.datetime :verified_at + t.boolean :success, null: false, default: false + t.string :domain, null: false + end + + add_index :email_address_verifications, :email, unique: true + add_index :email_address_verifications, :domain + end +end diff --git a/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb b/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb new file mode 100644 index 000000000..a7e2f8ee8 --- /dev/null +++ b/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb @@ -0,0 +1,13 @@ +class ChangeEmailVerificationFieldsToCitext < ActiveRecord::Migration[6.0] + def up + enable_extension 'citext' + change_column :email_address_verifications, :email, :citext + change_column :email_address_verifications, :domain, :citext + end + + def down + change_column :email_address_verifications, :email, :string + change_column :email_address_verifications, :domain, :string + disable_extension 'citext' + end +end diff --git a/db/structure.sql b/db/structure.sql index 2455bec5a..29e59a8a0 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -33,10 +33,17 @@ CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -- --- Name: EXTENSION btree_gist; Type: COMMENT; Schema: -; Owner: - +-- Name: citext; Type: EXTENSION; Schema: -; Owner: - -- -COMMENT ON EXTENSION btree_gist IS 'support for indexing common datatypes in GiST'; +CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public; + + +-- +-- Name: EXTENSION citext; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION citext IS 'data type for case-insensitive character strings'; -- @@ -817,7 +824,99 @@ ALTER SEQUENCE public.domains_id_seq OWNED BY public.domains.id; -- --- Name: epp_sessions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: email_address_verifications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_address_verifications ( + id bigint NOT NULL, + email public.citext NOT NULL, + verified_at timestamp without time zone, + success boolean DEFAULT false NOT NULL, + domain public.citext NOT NULL +); + + +-- +-- Name: email_address_verifications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_address_verifications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_address_verifications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_address_verifications_id_seq OWNED BY public.email_address_verifications.id; + + +-- +-- Name: email_addresses_validations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_addresses_validations ( + id bigint NOT NULL, + email character varying NOT NULL, + validated_at timestamp without time zone +); + + +-- +-- Name: email_addresses_validations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_addresses_validations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_addresses_validations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_addresses_validations_id_seq OWNED BY public.email_addresses_validations.id; + + +-- +-- Name: email_addresses_verifications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_addresses_verifications ( + id bigint NOT NULL, + email character varying NOT NULL, + validated_at timestamp without time zone +); + + +-- +-- Name: email_addresses_verifications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_addresses_verifications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_addresses_verifications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_addresses_verifications_id_seq OWNED BY public.email_addresses_verifications.id; + + +-- +-- Name: epp_sessions; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE public.epp_sessions ( @@ -2492,21 +2591,42 @@ ALTER TABLE ONLY public.domain_transfers ALTER COLUMN id SET DEFAULT nextval('pu -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: domains id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.domains ALTER COLUMN id SET DEFAULT nextval('public.domains_id_seq'::regclass); -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: email_address_verifications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_address_verifications ALTER COLUMN id SET DEFAULT nextval('public.email_address_verifications_id_seq'::regclass); + + +-- +-- Name: email_addresses_validations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_validations ALTER COLUMN id SET DEFAULT nextval('public.email_addresses_validations_id_seq'::regclass); + + +-- +-- Name: email_addresses_verifications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_verifications ALTER COLUMN id SET DEFAULT nextval('public.email_addresses_verifications_id_seq'::regclass); + + +-- +-- Name: epp_sessions id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.epp_sessions ALTER COLUMN id SET DEFAULT nextval('public.epp_sessions_id_seq'::regclass); -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: invoice_items id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.invoice_items ALTER COLUMN id SET DEFAULT nextval('public.invoice_items_id_seq'::regclass); @@ -2907,7 +3027,31 @@ ALTER TABLE ONLY public.domains -- --- Name: epp_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: email_address_verifications email_address_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_address_verifications + ADD CONSTRAINT email_address_verifications_pkey PRIMARY KEY (id); + + +-- +-- Name: email_addresses_validations email_addresses_validations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_validations + ADD CONSTRAINT email_addresses_validations_pkey PRIMARY KEY (id); + + +-- +-- Name: email_addresses_verifications email_addresses_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_verifications + ADD CONSTRAINT email_addresses_verifications_pkey PRIMARY KEY (id); + + +-- +-- Name: epp_sessions epp_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.epp_sessions @@ -2915,7 +3059,7 @@ ALTER TABLE ONLY public.epp_sessions -- --- Name: invoice_items_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: invoice_items invoice_items_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.invoice_items @@ -3494,434 +3638,447 @@ CREATE INDEX index_domains_on_registrar_id ON public.domains USING btree (regist -- --- Name: index_domains_on_statuses; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_domains_on_statuses; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_domains_on_statuses ON public.domains USING gin (statuses); -- --- Name: index_epp_sessions_on_updated_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_email_address_verifications_on_domain; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_email_address_verifications_on_domain ON public.email_address_verifications USING btree (domain); + + +-- +-- Name: index_epp_sessions_on_updated_at; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_epp_sessions_on_updated_at ON public.epp_sessions USING btree (updated_at); -- --- Name: index_invoice_items_on_invoice_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_invoice_items_on_invoice_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_invoice_items_on_invoice_id ON public.invoice_items USING btree (invoice_id); -- --- Name: index_invoices_on_buyer_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_invoices_on_buyer_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_invoices_on_buyer_id ON public.invoices USING btree (buyer_id); -- --- Name: index_legal_documents_on_checksum; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_legal_documents_on_checksum; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_legal_documents_on_checksum ON public.legal_documents USING btree (checksum); -- --- Name: index_legal_documents_on_documentable_type_and_documentable_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_legal_documents_on_documentable_type_and_documentable_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_legal_documents_on_documentable_type_and_documentable_id ON public.legal_documents USING btree (documentable_type, documentable_id); -- --- Name: index_log_account_activities_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_account_activities_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_account_activities_on_item_type_and_item_id ON public.log_account_activities USING btree (item_type, item_id); -- --- Name: index_log_account_activities_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_account_activities_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_account_activities_on_whodunnit ON public.log_account_activities USING btree (whodunnit); -- --- Name: index_log_accounts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_accounts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_accounts_on_item_type_and_item_id ON public.log_accounts USING btree (item_type, item_id); -- --- Name: index_log_accounts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_accounts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_accounts_on_whodunnit ON public.log_accounts USING btree (whodunnit); -- --- Name: index_log_bank_statements_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_statements_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_statements_on_item_type_and_item_id ON public.log_bank_statements USING btree (item_type, item_id); -- --- Name: index_log_bank_statements_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_statements_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_statements_on_whodunnit ON public.log_bank_statements USING btree (whodunnit); -- --- Name: index_log_bank_transactions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_transactions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_transactions_on_item_type_and_item_id ON public.log_bank_transactions USING btree (item_type, item_id); -- --- Name: index_log_bank_transactions_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_transactions_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_transactions_on_whodunnit ON public.log_bank_transactions USING btree (whodunnit); -- --- Name: index_log_blocked_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_blocked_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_blocked_domains_on_item_type_and_item_id ON public.log_blocked_domains USING btree (item_type, item_id); -- --- Name: index_log_blocked_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_blocked_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_blocked_domains_on_whodunnit ON public.log_blocked_domains USING btree (whodunnit); -- --- Name: index_log_certificates_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_certificates_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_certificates_on_item_type_and_item_id ON public.log_certificates USING btree (item_type, item_id); -- --- Name: index_log_certificates_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_certificates_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_certificates_on_whodunnit ON public.log_certificates USING btree (whodunnit); -- --- Name: index_log_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_contacts_on_item_type_and_item_id ON public.log_contacts USING btree (item_type, item_id); -- --- Name: index_log_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_contacts_on_whodunnit ON public.log_contacts USING btree (whodunnit); -- --- Name: index_log_dnskeys_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_dnskeys_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_dnskeys_on_item_type_and_item_id ON public.log_dnskeys USING btree (item_type, item_id); -- --- Name: index_log_dnskeys_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_dnskeys_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_dnskeys_on_whodunnit ON public.log_dnskeys USING btree (whodunnit); -- --- Name: index_log_domain_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domain_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domain_contacts_on_item_type_and_item_id ON public.log_domain_contacts USING btree (item_type, item_id); -- --- Name: index_log_domain_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domain_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contacts USING btree (whodunnit); -- --- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domains_on_item_type_and_item_id ON public.log_domains USING btree (item_type, item_id); -- --- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domains_on_whodunnit ON public.log_domains USING btree (whodunnit); -- --- Name: index_log_invoice_items_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoice_items_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoice_items_on_item_type_and_item_id ON public.log_invoice_items USING btree (item_type, item_id); -- --- Name: index_log_invoice_items_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoice_items_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoice_items_on_whodunnit ON public.log_invoice_items USING btree (whodunnit); -- --- Name: index_log_invoices_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoices_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoices_on_item_type_and_item_id ON public.log_invoices USING btree (item_type, item_id); -- --- Name: index_log_invoices_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoices_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoices_on_whodunnit ON public.log_invoices USING btree (whodunnit); -- --- Name: index_log_nameservers_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_nameservers_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_nameservers_on_item_type_and_item_id ON public.log_nameservers USING btree (item_type, item_id); -- --- Name: index_log_nameservers_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_nameservers_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_nameservers_on_whodunnit ON public.log_nameservers USING btree (whodunnit); -- --- Name: index_log_notifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_notifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_notifications_on_item_type_and_item_id ON public.log_notifications USING btree (item_type, item_id); -- --- Name: index_log_notifications_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_notifications_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_notifications_on_whodunnit ON public.log_notifications USING btree (whodunnit); -- --- Name: index_log_registrant_verifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrant_verifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrant_verifications_on_item_type_and_item_id ON public.log_registrant_verifications USING btree (item_type, item_id); -- --- Name: index_log_registrant_verifications_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrant_verifications_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrant_verifications_on_whodunnit ON public.log_registrant_verifications USING btree (whodunnit); -- --- Name: index_log_registrars_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrars_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrars_on_item_type_and_item_id ON public.log_registrars USING btree (item_type, item_id); -- --- Name: index_log_registrars_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrars_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrars_on_whodunnit ON public.log_registrars USING btree (whodunnit); -- --- Name: index_log_reserved_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_reserved_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_reserved_domains_on_item_type_and_item_id ON public.log_reserved_domains USING btree (item_type, item_id); -- --- Name: index_log_reserved_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_reserved_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_reserved_domains_on_whodunnit ON public.log_reserved_domains USING btree (whodunnit); -- --- Name: index_log_settings_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_settings_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_settings_on_item_type_and_item_id ON public.log_settings USING btree (item_type, item_id); -- --- Name: index_log_settings_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_settings_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_settings_on_whodunnit ON public.log_settings USING btree (whodunnit); -- --- Name: index_log_users_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_users_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_users_on_item_type_and_item_id ON public.log_users USING btree (item_type, item_id); -- --- Name: index_log_users_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_users_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_users_on_whodunnit ON public.log_users USING btree (whodunnit); -- --- Name: index_nameservers_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_nameservers_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_nameservers_on_domain_id ON public.nameservers USING btree (domain_id); -- --- Name: index_notifications_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_notifications_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_notifications_on_registrar_id ON public.notifications USING btree (registrar_id); -- --- Name: index_payment_orders_on_invoice_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_payment_orders_on_invoice_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_payment_orders_on_invoice_id ON public.payment_orders USING btree (invoice_id); -- --- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_prices_on_zone_id ON public.prices USING btree (zone_id); -- --- Name: index_registrant_verifications_on_created_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_registrant_verifications_on_created_at; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_registrant_verifications_on_created_at ON public.registrant_verifications USING btree (created_at); -- --- Name: index_registrant_verifications_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_registrant_verifications_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_registrant_verifications_on_domain_id ON public.registrant_verifications USING btree (domain_id); -- --- Name: index_settings_on_thing_type_and_thing_id_and_var; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_settings_on_thing_type_and_thing_id_and_var; Type: INDEX; Schema: public; Owner: - -- CREATE UNIQUE INDEX index_settings_on_thing_type_and_thing_id_and_var ON public.settings USING btree (thing_type, thing_id, var); -- --- Name: index_users_on_identity_code; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_users_on_identity_code; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_users_on_identity_code ON public.users USING btree (identity_code); -- --- Name: index_users_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_users_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_id); -- --- Name: index_versions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_versions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_versions_on_item_type_and_item_id ON public.versions USING btree (item_type, item_id); -- --- Name: index_whois_records_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_whois_records_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_whois_records_on_domain_id ON public.whois_records USING btree (domain_id); -- --- Name: index_whois_records_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_whois_records_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_whois_records_on_registrar_id ON public.whois_records USING btree (registrar_id); -- --- Name: log_contacts_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_contacts_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_contacts_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_id'::text))::integer)); -- --- Name: log_dnskeys_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_dnskeys_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_dnskeys_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_domain_id'::text))::integer)); -- --- Name: log_domains_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_domains_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_domains_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_id'::text))::integer)); -- --- Name: log_nameservers_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_nameservers_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_nameservers_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_domain_id'::text))::integer)); -- --- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: unique_data_migrations; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX unique_data_migrations ON public.data_migrations USING btree (version); + + +-- +-- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: - -- CREATE UNIQUE INDEX unique_schema_migrations ON public.schema_migrations USING btree (version); - -- --- Name: contacts_registrar_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: contacts contacts_registrar_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.contacts @@ -3929,7 +4086,7 @@ ALTER TABLE ONLY public.contacts -- --- Name: domain_contacts_contact_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: domain_contacts domain_contacts_contact_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.domain_contacts @@ -4551,7 +4708,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20200505150413'), ('20200518104105'), ('20200529115011'), +('20200605100827'), +('20200610090110'), ('20200630081231'), ('20200714115338'); - diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake new file mode 100644 index 000000000..d49bb38b9 --- /dev/null +++ b/lib/tasks/verify_email.rake @@ -0,0 +1,23 @@ +namespace :verify_email do + desc 'Stars verifying email jobs for all the domain' + task all_domains: :environment do + verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) + verifications_by_domain.each do |_domain, verifications| + ver = verifications.sample # Verify random email to not to clog the SMTP servers + VerifyEmailsJob.enqueue(ver.id) + next + end + end + + # Need to be run like 'bundle exec rake verify_email:domain['gmail.com']' + # In zsh syntax will be 'bundle exec rake verify_email:domain\['gmail.com'\]' + # Default 'bundle exec rake verify_email:domain' wil use 'internet.ee' domain + desc 'Stars verifying email jobs for domain stated in argument' + task :domain, [:domain_name] => [:environment] do |_task, args| + args.with_defaults(domain_name: 'internet.ee') + + verifications_by_domain = EmailAddressVerification.not_verified_recently + .by_domain(args[:domain_name]) + verifications_by_domain.map { |ver| VerifyEmailsJob.enqueue(ver.id) } + end +end diff --git a/test/fixtures/contacts.yml b/test/fixtures/contacts.yml index ddfbfe93f..0173d56dd 100644 --- a/test/fixtures/contacts.yml +++ b/test/fixtures/contacts.yml @@ -85,3 +85,11 @@ invalid: auth_info: any registrar: bestnames uuid: bd80c0f9-26ee-49e0-a2cb-2311d931c433 + +invalid_email: + name: any + code: invalid_email + email: invalid@invalid. + auth_info: any + registrar: bestnames + uuid: fa8c4f51-a221-4628-b3c6-47995f4edea3 diff --git a/test/jobs/verify_emails_job_test.rb b/test/jobs/verify_emails_job_test.rb new file mode 100644 index 000000000..f55a474db --- /dev/null +++ b/test/jobs/verify_emails_job_test.rb @@ -0,0 +1,59 @@ +require "test_helper" + +class VerifyEmailsJobTest < ActiveSupport::TestCase + def setup + @contact = contacts(:john) + @invalid_contact = contacts(:invalid_email) + @contact_verification = @contact.email_verification + @invalid_contact_verification = @invalid_contact.email_verification + + @default_whitelist = Truemail.configure.whitelisted_domains + @default_blacklist = Truemail.configure.blacklisted_domains + Truemail.configure.whitelisted_domains = whitelisted_domains + Truemail.configure.blacklisted_domains = blacklisted_domains + end + + def teardown + Truemail.configure.whitelisted_domains = @default_whitelist + Truemail.configure.blacklisted_domains = @default_blacklist + end + + def domain(email) + Mail::Address.new(email).domain + rescue Mail::Field::IncompleteParseError + nil + end + + def whitelisted_domains + [domain(@contact.email)].reject(&:blank?) + end + + def blacklisted_domains + [domain(@invalid_contact.email)].reject(&:blank?) + end + + def test_job_checks_if_email_valid + VerifyEmailsJob.run(@contact_verification.id) + @contact_verification.reload + + assert @contact_verification.success + end + + def test_job_checks_does_not_run_if_recent + old_verified_at = Time.zone.now - 10.days + @contact_verification.update(success: true, verified_at: old_verified_at) + assert @contact_verification.recently_verified? + + VerifyEmailsJob.run(@contact_verification.id) + @contact_verification.reload + + assert_in_delta @contact_verification.verified_at.to_i, old_verified_at.to_i, 1 + end + + def test_job_checks_if_email_invalid + VerifyEmailsJob.run(@invalid_contact_verification.id) + @contact_verification.reload + + refute @contact_verification.success + end +end diff --git a/test/models/contact_test.rb b/test/models/contact_test.rb index f71546088..f833011c6 100644 --- a/test/models/contact_test.rb +++ b/test/models/contact_test.rb @@ -3,6 +3,11 @@ require 'test_helper' class ContactTest < ActiveSupport::TestCase setup do @contact = contacts(:john) + @old_validation_type = Truemail.configure.default_validation_type + end + + teardown do + Truemail.configure.default_validation_type = @old_validation_type end def test_valid_contact_fixture_is_valid @@ -61,10 +66,17 @@ class ContactTest < ActiveSupport::TestCase assert contact.invalid? end - def test_validates_email_format + def test_email_verification_valid contact = valid_contact + contact.email = 'info@internet.ee' + assert contact.valid? + end - contact.email = 'invalid' + def test_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + contact = valid_contact + contact.email = 'somecrude1337joke@internet.ee' assert contact.invalid? assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), contact.errors.messages[:email].first end @@ -273,6 +285,16 @@ class ContactTest < ActiveSupport::TestCase assert_equal domain.whois_record.try(:json).try(:[], 'registrant'), @contact.name end + def test_creates_email_verification_in_unicode + unicode_email = 'suur@äri.ee' + punycode_email = Contact.unicode_to_punycode(unicode_email) + + @contact.email = punycode_email + @contact.save + + assert_equal @contact.email_verification.email, unicode_email + end + private def make_contact_free_of_domains_where_it_acts_as_a_registrant(contact) diff --git a/test/models/registrar_test.rb b/test/models/registrar_test.rb index 2536bbdd8..091b8e6f4 100644 --- a/test/models/registrar_test.rb +++ b/test/models/registrar_test.rb @@ -5,11 +5,13 @@ class RegistrarTest < ActiveSupport::TestCase @registrar = registrars(:bestnames) @original_default_language = Setting.default_language @original_days_to_keep_invoices_active = Setting.days_to_keep_invoices_active + @old_validation_type = Truemail.configure.default_validation_type end teardown do Setting.default_language = @original_default_language Setting.days_to_keep_invoices_active = @original_days_to_keep_invoices_active + Truemail.configure.default_validation_type = @old_validation_type end def test_valid_registrar_is_valid @@ -38,10 +40,21 @@ class RegistrarTest < ActiveSupport::TestCase assert registrar.invalid? end - def test_email_format_validation + def test_email_verification_valid registrar = valid_registrar + registrar.email = 'info@internet.ee' + registrar.billing_email = nil + + assert registrar.valid? + end + + def test_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + registrar = valid_registrar + registrar.email = 'somecrude1337joke@internet.ee' + registrar.billing_email = nil - registrar.email = 'invalid' assert registrar.invalid? assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), registrar.errors.messages[:email].first end @@ -57,23 +70,30 @@ class RegistrarTest < ActiveSupport::TestCase assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_mx_check_error'), registrar.errors.messages[:email].first end - def test_invalid_without_accounting_customer_code + def test_email_verification_regex_error + Truemail.configure.default_validation_type = :regex + registrar = valid_registrar - registrar.accounting_customer_code = '' + registrar.email = 'some@strangesentence@internet.ee' + registrar.billing_email = nil + assert registrar.invalid? assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error'), registrar.errors.messages[:email].first end - def test_optional_billing_email + def test_billing_email_verification_valid registrar = valid_registrar - registrar.billing_email = '' + registrar.billing_email = 'info@internet.ee' + assert registrar.valid? end - def test_billing_email_format_validation - registrar = valid_registrar + def test_billing_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + registrar = valid_registrar + registrar.billing_email = 'somecrude1337joke@internet.ee' - registrar.billing_email = 'invalid' assert registrar.invalid? assert_equal I18n.t('activerecord.errors.models.contact.attributes.email.email_smtp_check_error'), registrar.errors.messages[:billing_email].first end diff --git a/test/tasks/emails/verify_email_task_test.rb b/test/tasks/emails/verify_email_task_test.rb new file mode 100644 index 000000000..7cca11845 --- /dev/null +++ b/test/tasks/emails/verify_email_task_test.rb @@ -0,0 +1,63 @@ +require 'test_helper' + +class VerifyEmailTaskTest < ActiveSupport::TestCase + + def setup + @contact = contacts(:john) + @invalid_contact = contacts(:invalid_email) + @contact_verification = @contact.email_verification + @invalid_contact_verification = @invalid_contact.email_verification + + @default_whitelist = Truemail.configure.whitelisted_domains + @default_blacklist = Truemail.configure.blacklisted_domains + Truemail.configure.whitelisted_domains = whitelisted_domains + Truemail.configure.blacklisted_domains = blacklisted_domains + end + + def teardown + Truemail.configure.whitelisted_domains = @default_whitelist + Truemail.configure.blacklisted_domains = @default_blacklist + end + + def domain(email) + Mail::Address.new(email).domain + rescue Mail::Field::IncompleteParseError + nil + end + + def whitelisted_domains + [domain(@contact.email)].reject(&:blank?) + end + + def blacklisted_domains + [domain(@invalid_contact.email)].reject(&:blank?) + end + + def test_tasks_verifies_emails + capture_io { run_task } + + @contact_verification.reload + @invalid_contact_verification.reload + + assert @contact_verification.verified? + assert @invalid_contact_verification.failed? + end + + def test_domain_task_verifies_for_one_domain + capture_io { run_single_domain_task(@contact_verification.domain) } + + @contact_verification.reload + @invalid_contact_verification.reload + + assert @contact_verification.verified? + assert @invalid_contact_verification.not_verified? + end + + def run_task + Rake::Task['verify_email:all_domains'].execute + end + + def run_single_domain_task(domain) + Rake::Task["verify_email:domain"].invoke(domain) + end +end