From 2d89e8124f0388489ac384723784fe791d5cc724 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 3 Jun 2020 16:40:13 +0500 Subject: [PATCH 01/26] Add gem Truemail Closes # --- Gemfile | 3 +- Gemfile.lock | 8 +++- config/initializers/truemail.rb | 71 +++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 config/initializers/truemail.rb diff --git a/Gemfile b/Gemfile index f6c8a6397..46f507614 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,7 @@ gem 'pg', '1.2.2' # 1.8 is for Rails < 5.0 gem 'ransack', '~> 2.3' gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and RFC 3696 +gem 'truemail', '~> 1.7' # validates email by regexp, mail server existence and address existence # 0.7.3 is the latest for Rails 4.2, however, it is absent on Rubygems server # https://github.com/huacnlee/rails-settings-cached/issues/165 @@ -39,7 +40,7 @@ gem 'grape' # registry specfic gem 'data_migrate', '~> 6.1' gem 'isikukood' # for EE-id validation -gem 'simpleidn', '0.0.9' # For punycode +gem 'simpleidn', '0.1.1' # For punycode gem 'money-rails' gem 'whenever', '0.9.4', require: false diff --git a/Gemfile.lock b/Gemfile.lock index effb7ff38..dde0effa4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -434,7 +434,8 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - simpleidn (0.0.9) + simpleidn (0.1.1) + unf (~> 0.1.4) sinatra (2.0.8.1) mustermann (~> 1.0) rack (~> 2.0) @@ -454,6 +455,8 @@ GEM thor (0.20.3) thread_safe (0.3.6) tilt (2.0.10) + truemail (1.7.1) + simpleidn (~> 0.1.1) tzinfo (1.2.7) thread_safe (~> 0.1) uglifier (4.2.0) @@ -542,7 +545,8 @@ DEPENDENCIES select2-rails (= 3.5.9.3) selectize-rails (= 0.12.1) simplecov (= 0.17.1) - simpleidn (= 0.0.9) + simpleidn (= 0.1.1) + truemail (~> 1.7) uglifier validates_email_format_of (= 1.6.3) webdrivers diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb new file mode 100644 index 000000000..f43b73479 --- /dev/null +++ b/config/initializers/truemail.rb @@ -0,0 +1,71 @@ +require 'truemail' + +Truemail.configure do |config| + # Required parameter. Must be an existing email on behalf of which verification will be performed + config.verifier_email = 'info@internet.ee' + + # Optional parameter. Must be an existing domain on behalf of which verification will be performed. + # By default verifier domain based on verifier email + # config.verifier_domain = 'internet.ee' + + # Optional parameter. You can override default regex pattern + # config.email_pattern = /regex_pattern/ + + # Optional parameter. You can override default regex pattern + # config.smtp_error_body_pattern = /regex_pattern/ + + # Optional parameter. Connection timeout is equal to 2 ms by default. + # config.connection_timeout = 1 + + # Optional parameter. A SMTP server response timeout is equal to 2 ms by default. + # config.response_timeout = 1 + + # Optional parameter. Total of connection attempts. It is equal to 2 by default. + # This parameter uses in mx lookup timeout error and smtp request (for cases when + # there is one mx server). + config.connection_attempts = 3 + + # Optional parameter. You can predefine default validation type for + # Truemail.validate('email@email.com') call without with-parameter + # Available validation types: :regex, :mx, :smtp + config.default_validation_type = :smtp + + # Optional parameter. You can predefine which type of validation will be used for domains. + # Also you can skip validation by domain. Available validation types: :regex, :mx, :smtp + # This configuration will be used over current or default validation type parameter + # All of validations for 'somedomain.com' will be processed with regex validation only. + # And all of validations for 'otherdomain.com' will be processed with mx validation only. + # It is equal to empty hash by default. + # config.validation_type_for = { 'somedomain.com' => :regex, 'otherdomain.com' => :mx } + + # Optional parameter. Validation of email which contains whitelisted domain always will + # return true. Other validations will not processed even if it was defined in validation_type_for + # It is equal to empty array by default. + # config.whitelisted_domains = [] + + # Optional parameter. With this option Truemail will validate email which contains whitelisted + # domain only, i.e. if domain whitelisted, validation will passed to Regex, MX or SMTP validators. + # Validation of email which not contains whitelisted domain always will return false. + # It is equal false by default. + #config.whitelist_validation = true + + # Optional parameter. Validation of email which contains blacklisted domain always will + # return false. Other validations will not processed even if it was defined in validation_type_for + # It is equal to empty array by default. + #config.blacklisted_domains = [] + + # Optional parameter. This option will provide to use not RFC MX lookup flow. + # It means that MX and Null MX records will be cheked on the DNS validation layer only. + # By default this option is disabled. + # config.not_rfc_mx_lookup_flow = true + + # Optional parameter. This option will be parse bodies of SMTP errors. It will be helpful + # if SMTP server does not return an exact answer that the email does not exist + # By default this option is disabled, available for SMTP validation only. + # config.smtp_safe_check = true + + # Optional parameter. This option will enable tracking events. You can print tracking events to + # stdout, write to file or both of these. Tracking event by default is :error + # Available tracking event: :all, :unrecognized_error, :recognized_error, :error + # config.logger = { tracking_event: :all, stdout: true, log_absolute_path: '/home/app/log/truemail.log' } +end From ba4e256662ad2f45319ab7c594f67fa227311237 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 3 Jun 2020 17:07:19 +0500 Subject: [PATCH 02/26] Add test domains whitelist & fix tests --- Gemfile | 3 ++- app/models/nameserver.rb | 2 +- config/initializers/truemail.rb | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 46f507614..050c95ba0 100644 --- a/Gemfile +++ b/Gemfile @@ -15,8 +15,9 @@ gem 'paper_trail', '~> 10.3' gem 'pg', '1.2.2' # 1.8 is for Rails < 5.0 gem 'ransack', '~> 2.3' -gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and RFC 3696 gem 'truemail', '~> 1.7' # validates email by regexp, mail server existence and address existence +gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and RFC 3696 + # 0.7.3 is the latest for Rails 4.2, however, it is absent on Rubygems server # https://github.com/huacnlee/rails-settings-cached/issues/165 diff --git a/app/models/nameserver.rb b/app/models/nameserver.rb index 3ddf1d1c5..3e4051165 100644 --- a/app/models/nameserver.rb +++ b/app/models/nameserver.rb @@ -88,7 +88,7 @@ class Nameserver < ApplicationRecord end def normalize_attributes - self.hostname = hostname.try(:strip).try(:downcase) + self.hostname = hostname.try(:strip).try(:downcase).gsub(/\.$/, '') self.ipv4 = Array(ipv4).reject(&:blank?).map(&:strip) self.ipv6 = Array(ipv6).reject(&:blank?).map(&:strip).map(&:upcase) end diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb index f43b73479..efcecd508 100644 --- a/config/initializers/truemail.rb +++ b/config/initializers/truemail.rb @@ -43,6 +43,11 @@ Truemail.configure do |config| # It is equal to empty array by default. # config.whitelisted_domains = [] + unless Rails.env.production? + config.whitelisted_domains = %w[bestnames.test goodnames.test example.com inbox.test mail.test + outlook.test invalid.test email.test] + end + # Optional parameter. With this option Truemail will validate email which contains whitelisted # domain only, i.e. if domain whitelisted, validation will passed to Regex, MX or SMTP validators. # Validation of email which not contains whitelisted domain always will return false. From e56162f37db1605a9e7cab123e57a2fc3342b3c4 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 3 Jun 2020 17:25:33 +0500 Subject: [PATCH 03/26] Add registrar email validation --- Gemfile | 1 - app/models/registrar.rb | 21 ++++++++++++++++----- config/initializers/truemail.rb | 11 +++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/Gemfile b/Gemfile index 050c95ba0..0804ca4db 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,6 @@ gem 'ransack', '~> 2.3' gem 'truemail', '~> 1.7' # validates email by regexp, mail server existence and address existence gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and RFC 3696 - # 0.7.3 is the latest for Rails 4.2, however, it is absent on Rubygems server # https://github.com/huacnlee/rails-settings-cached/issues/165 gem 'rails-settings-cached', '0.7.2' diff --git a/app/models/registrar.rb b/app/models/registrar.rb index dbdd7e8d3..2e2051542 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -29,14 +29,11 @@ class Registrar < ApplicationRecord validates :vat_rate, numericality: { greater_than_or_equal_to: 0, less_than: 100 }, allow_nil: true - validate :forbid_special_code - attribute :vat_rate, ::Type::VATRate.new after_initialize :set_defaults - validates :email, email_format: { message: :invalid }, - allow_blank: true, if: proc { |c| c.will_save_change_to_email? } - validates :billing_email, email_format: { message: :invalid }, allow_blank: true + validate :correct_email_format, if: proc { |c| c.will_save_change_to_email? } + validate :correct_billing_email_format alias_attribute :contact_email, :email @@ -196,4 +193,18 @@ class Registrar < ApplicationRecord def vat_liable_in_foreign_country? !vat_liable_locally? end + + def verify_email_mx_smtp(field:, email:) + errors.add(field, :invalid) unless email.blank? || Truemail.valid?(email) + end + + def correct_email_format + verify_email_mx_smtp(field: :email, email: email) + end + + def correct_billing_email_format + return if self[:billing_email].blank? + + verify_email_mx_smtp(field: :billing_email, email: billing_email) + end end diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb index efcecd508..337ac8605 100644 --- a/config/initializers/truemail.rb +++ b/config/initializers/truemail.rb @@ -28,7 +28,11 @@ Truemail.configure do |config| # Optional parameter. You can predefine default validation type for # Truemail.validate('email@email.com') call without with-parameter # Available validation types: :regex, :mx, :smtp - config.default_validation_type = :smtp + if Rails.env.production? + config.default_validation_type = :smtp + else + config.default_validation_type = :regex + end # Optional parameter. You can predefine which type of validation will be used for domains. # Also you can skip validation by domain. Available validation types: :regex, :mx, :smtp @@ -43,11 +47,6 @@ Truemail.configure do |config| # It is equal to empty array by default. # config.whitelisted_domains = [] - unless Rails.env.production? - config.whitelisted_domains = %w[bestnames.test goodnames.test example.com inbox.test mail.test - outlook.test invalid.test email.test] - end - # Optional parameter. With this option Truemail will validate email which contains whitelisted # domain only, i.e. if domain whitelisted, validation will passed to Regex, MX or SMTP validators. # Validation of email which not contains whitelisted domain always will return false. From 1b06e43f18e166a1be865cb078ef7d418d763d4f Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Thu, 4 Jun 2020 14:11:50 +0500 Subject: [PATCH 04/26] Move email checks and validation methods to concern --- app/models/concerns/email_checkable.rb | 19 +++++++++++++++++++ app/models/registrar.rb | 15 +-------------- 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 app/models/concerns/email_checkable.rb diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_checkable.rb new file mode 100644 index 000000000..193352d74 --- /dev/null +++ b/app/models/concerns/email_checkable.rb @@ -0,0 +1,19 @@ +module Concerns + module EmailCheckable + extend ActiveSupport::Concern + + def verify_email_mx_smtp(field:, email:) + errors.add(field, :invalid) unless email.blank? || Truemail.valid?(email) + end + + def correct_email_format + verify_email_mx_smtp(field: :email, email: email) + end + + def correct_billing_email_format + return if self[:billing_email].blank? + + verify_email_mx_smtp(field: :billing_email, email: billing_email) + end + end +end diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 2e2051542..6e6d41b77 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -1,6 +1,7 @@ class Registrar < ApplicationRecord include Versions # version/registrar_version.rb include Concerns::Registrar::BookKeeping + include Concerns::EmailCheckable include Concerns::Registrar::LegalDoc has_many :domains, dependent: :restrict_with_error @@ -193,18 +194,4 @@ class Registrar < ApplicationRecord def vat_liable_in_foreign_country? !vat_liable_locally? end - - def verify_email_mx_smtp(field:, email:) - errors.add(field, :invalid) unless email.blank? || Truemail.valid?(email) - end - - def correct_email_format - verify_email_mx_smtp(field: :email, email: email) - end - - def correct_billing_email_format - return if self[:billing_email].blank? - - verify_email_mx_smtp(field: :billing_email, email: billing_email) - end end From 9f5a636d726a199a34ecaf17fd092d114e04d3c5 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Thu, 4 Jun 2020 14:47:12 +0500 Subject: [PATCH 05/26] Change contact email validation --- app/models/contact.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/contact.rb b/app/models/contact.rb index f07c0c114..4b60e85be 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -7,6 +7,7 @@ class Contact < ApplicationRecord include Concerns::Contact::Transferable include Concerns::Contact::Identical include Concerns::Contact::Disclosable + include Concerns::EmailCheckable belongs_to :original, class_name: self.name belongs_to :registrar, required: true @@ -29,8 +30,7 @@ class Contact < ApplicationRecord validates :phone, presence: true, e164: true, phone: true - validates :email, format: /@/ - validates :email, email_format: { message: :invalid }, if: proc { |c| c.will_save_change_to_email? } + validate :correct_email_format, if: proc { |c| c.will_save_change_to_email? } validates :code, uniqueness: { message: :epp_id_taken }, From b8ade7628d62f9400999b0b62d91107573c511e1 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Thu, 4 Jun 2020 15:35:09 +0500 Subject: [PATCH 06/26] Add tests to see if Truemail validates ok --- test/models/contact_test.rb | 23 ++++++++++++++---- test/models/registrar_test.rb | 46 +++++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/test/models/contact_test.rb b/test/models/contact_test.rb index 4e500e98d..77e959a95 100644 --- a/test/models/contact_test.rb +++ b/test/models/contact_test.rb @@ -61,14 +61,27 @@ class ContactTest < ActiveSupport::TestCase assert contact.invalid? end - def test_validates_email_format - contact = valid_contact + def tests_email_mx_and_smtp + Truemail.configure do |config| + config.default_validation_type = :smtp + end - contact.email = 'invalid' + contact = valid_contact + contact.email = 'info@internet.ee' + assert contact.valid? + + contact.email = 'somecrude1337joke@internet.ee' assert contact.invalid? - contact.email = 'valid@registrar.test' - assert contact.valid? + contact.email = 'some@strangesentence@internet.ee' + assert contact.invalid? + + contact.email = 'somecrude31337joke@somestrange31337domain.ee' + assert contact.invalid? + + Truemail.configure do |config| + config.default_validation_type = :regex + end end def test_invalid_without_phone diff --git a/test/models/registrar_test.rb b/test/models/registrar_test.rb index c5d832922..6db466e1c 100644 --- a/test/models/registrar_test.rb +++ b/test/models/registrar_test.rb @@ -38,14 +38,27 @@ class RegistrarTest < ActiveSupport::TestCase assert registrar.invalid? end - def test_email_format_validation - registrar = valid_registrar + def tests_email_mx_and_smtp + Truemail.configure do |config| + config.default_validation_type = :smtp + end - registrar.email = 'invalid' + registrar = valid_registrar + registrar.email = 'info@internet.ee' + assert registrar.valid? + + registrar.email = 'somecrude1337joke@internet.ee' assert registrar.invalid? - registrar.email = 'valid@email.test' - assert registrar.valid? + registrar.email = 'some@strangesentence@internet.ee' + assert registrar.invalid? + + registrar.email = 'somecrude31337joke@somestrange31337domain.ee' + assert registrar.invalid? + + Truemail.configure do |config| + config.default_validation_type = :regex + end end def test_invalid_without_accounting_customer_code @@ -60,14 +73,27 @@ class RegistrarTest < ActiveSupport::TestCase assert registrar.valid? end - def test_billing_email_format_validation - registrar = valid_registrar + def tests_email_mx_and_smtp + Truemail.configure do |config| + config.default_validation_type = :smtp + end - registrar.billing_email = 'invalid' + registrar = valid_registrar + registrar.billing_email = 'info@internet.ee' + assert registrar.valid? + + registrar.billing_email = 'somecrude1337joke@internet.ee' assert registrar.invalid? - registrar.billing_email = 'valid@email.test' - assert registrar.valid? + registrar.billing_email = 'непонятное@словосочетание@internet.ee' + assert registrar.invalid? + + registrar.billing_email = 'somecrude31337joke@somestrange31337domain.ee' + assert registrar.invalid? + + Truemail.configure do |config| + config.default_validation_type = :regex + end end def test_returns_billing_email_when_provided From cc142076c30e4757943d7cca8b2e5a7d96d22818 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Fri, 5 Jun 2020 16:41:56 +0500 Subject: [PATCH 07/26] Add verification model & migration --- app/models/concerns/email_checkable.rb | 12 ++ app/models/email_address_verification.rb | 4 + ...0827_create_email_address_verifications.rb | 10 ++ db/structure.sql | 149 +++++++++++++++++- 4 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 app/models/email_address_verification.rb create mode 100644 db/migrate/20200605100827_create_email_address_verifications.rb diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_checkable.rb index 193352d74..67975e65a 100644 --- a/app/models/concerns/email_checkable.rb +++ b/app/models/concerns/email_checkable.rb @@ -2,6 +2,18 @@ module Concerns module EmailCheckable extend ActiveSupport::Concern + def email_verification + EmailAddressVerification.find_by(email: self.email) + end + + def billing_email_verification + if self.attribute_names.include?('billing_email') + EmailAddressVerification.find_by(email: self.billing_email) + else + nil + end + end + def verify_email_mx_smtp(field:, email:) errors.add(field, :invalid) unless email.blank? || Truemail.valid?(email) end diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb new file mode 100644 index 000000000..15544e6e4 --- /dev/null +++ b/app/models/email_address_verification.rb @@ -0,0 +1,4 @@ +class EmailAddressVerification < ApplicationRecord + + +end diff --git a/db/migrate/20200605100827_create_email_address_verifications.rb b/db/migrate/20200605100827_create_email_address_verifications.rb new file mode 100644 index 000000000..39f59b51e --- /dev/null +++ b/db/migrate/20200605100827_create_email_address_verifications.rb @@ -0,0 +1,10 @@ +class CreateEmailAddressVerifications < ActiveRecord::Migration[6.0] + def change + create_table :email_address_verifications do |t| + t.string :email, null: false + t.datetime :verified_at + end + + add_index :email_address_verifications, :email, unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index 960481c44..239370878 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -817,7 +817,97 @@ ALTER SEQUENCE public.domains_id_seq OWNED BY public.domains.id; -- --- Name: epp_sessions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: email_address_verifications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_address_verifications ( + id bigint NOT NULL, + email character varying NOT NULL, + verified_at timestamp without time zone +); + + +-- +-- Name: email_address_verifications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_address_verifications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_address_verifications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_address_verifications_id_seq OWNED BY public.email_address_verifications.id; + + +-- +-- Name: email_addresses_validations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_addresses_validations ( + id bigint NOT NULL, + email character varying NOT NULL, + validated_at timestamp without time zone +); + + +-- +-- Name: email_addresses_validations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_addresses_validations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_addresses_validations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_addresses_validations_id_seq OWNED BY public.email_addresses_validations.id; + + +-- +-- Name: email_addresses_verifications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_addresses_verifications ( + id bigint NOT NULL, + email character varying NOT NULL, + validated_at timestamp without time zone +); + + +-- +-- Name: email_addresses_verifications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.email_addresses_verifications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: email_addresses_verifications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.email_addresses_verifications_id_seq OWNED BY public.email_addresses_verifications.id; + + +-- +-- Name: epp_sessions; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE public.epp_sessions ( @@ -2492,21 +2582,42 @@ ALTER TABLE ONLY public.domain_transfers ALTER COLUMN id SET DEFAULT nextval('pu -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: domains id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.domains ALTER COLUMN id SET DEFAULT nextval('public.domains_id_seq'::regclass); -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: email_address_verifications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_address_verifications ALTER COLUMN id SET DEFAULT nextval('public.email_address_verifications_id_seq'::regclass); + + +-- +-- Name: email_addresses_validations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_validations ALTER COLUMN id SET DEFAULT nextval('public.email_addresses_validations_id_seq'::regclass); + + +-- +-- Name: email_addresses_verifications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_verifications ALTER COLUMN id SET DEFAULT nextval('public.email_addresses_verifications_id_seq'::regclass); + + +-- +-- Name: epp_sessions id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.epp_sessions ALTER COLUMN id SET DEFAULT nextval('public.epp_sessions_id_seq'::regclass); -- --- Name: id; Type: DEFAULT; Schema: public; Owner: - +-- Name: invoice_items id; Type: DEFAULT; Schema: public; Owner: - -- ALTER TABLE ONLY public.invoice_items ALTER COLUMN id SET DEFAULT nextval('public.invoice_items_id_seq'::regclass); @@ -2907,7 +3018,31 @@ ALTER TABLE ONLY public.domains -- --- Name: epp_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: email_address_verifications email_address_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_address_verifications + ADD CONSTRAINT email_address_verifications_pkey PRIMARY KEY (id); + + +-- +-- Name: email_addresses_validations email_addresses_validations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_validations + ADD CONSTRAINT email_addresses_validations_pkey PRIMARY KEY (id); + + +-- +-- Name: email_addresses_verifications email_addresses_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_addresses_verifications + ADD CONSTRAINT email_addresses_verifications_pkey PRIMARY KEY (id); + + +-- +-- Name: epp_sessions epp_sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.epp_sessions @@ -2915,7 +3050,7 @@ ALTER TABLE ONLY public.epp_sessions -- --- Name: invoice_items_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: invoice_items invoice_items_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.invoice_items @@ -4535,6 +4670,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20200505150413'), ('20200518104105'), ('20200529115011'), +('20200605100827'), ('20200630081231'); - From 9ae85f3d744dc17d3c9f35990cc2fec61bc7a33b Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Fri, 5 Jun 2020 17:21:43 +0500 Subject: [PATCH 08/26] Add verification methods --- app/models/concerns/email_checkable.rb | 4 ++-- app/models/email_address_verification.rb | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_checkable.rb index 67975e65a..3edb9614d 100644 --- a/app/models/concerns/email_checkable.rb +++ b/app/models/concerns/email_checkable.rb @@ -3,12 +3,12 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_by(email: self.email) + EmailAddressVerification.find_or_create_by(email: self.email) end def billing_email_verification if self.attribute_names.include?('billing_email') - EmailAddressVerification.find_by(email: self.billing_email) + EmailAddressVerification.find_or_create_by(email: self.billing_email) else nil end diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 15544e6e4..4fa523764 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -1,4 +1,14 @@ class EmailAddressVerification < ApplicationRecord + RECENTLY_VERIFIED_PERIOD = 1.month + def recently_verified? + verified_at > Time.zone.now - RECENTLY_VERIFIED_PERIOD + end + + def verify + validation_request = Truemail.validate(email) + + update(verified_at: Time.zone.now) if validation_request.result.success + end end From 95a017629eabdf7b19b2787ebc57e411b1a202a5 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Fri, 5 Jun 2020 17:36:37 +0500 Subject: [PATCH 09/26] Add email verification job --- app/jobs/verify_emails_job.rb | 43 ++++++++++++++++++++++++ app/models/email_address_verification.rb | 3 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 app/jobs/verify_emails_job.rb diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb new file mode 100644 index 000000000..4aedddb94 --- /dev/null +++ b/app/jobs/verify_emails_job.rb @@ -0,0 +1,43 @@ +class SendEInvoiceJob < Que::Job + + def run(verification_id) + email_address_verification = run_condition(EmailAddressVerification + .find(run_condition(verification_id))) + + ActiveRecord::Base.transaction do + email_address_verification.verify + log_success(email_address_verification) + destroy + end + rescue StandardError => e + log_error(verification: email_address_verification, error: e) + raise e + end + + private + + def run_condition(email_address_verification) + destroy if email_address_verification.recently_verified? + email_address_verification + end + + def logger + Rails.logger + end + + def log_success(verification) + email = verification.try(:email) || verification + message = "Email address #{email} verified successfully" + logger.info message + end + + def log_error(verification:, error:) + email = verification.try(:email) || verification + message = <<~TEXT.squish + There was an error verifying email #{email}. + The error message was the following: #{error} + This job will retry. + TEXT + logger.error message + end +end diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 4fa523764..8afc74a78 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -3,7 +3,8 @@ class EmailAddressVerification < ApplicationRecord RECENTLY_VERIFIED_PERIOD = 1.month def recently_verified? - verified_at > Time.zone.now - RECENTLY_VERIFIED_PERIOD + verified_at.present? && + verified_at > Time.zone.now - RECENTLY_VERIFIED_PERIOD end def verify From 9758c822215f5718f14b890ab1f88720488d6bd6 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 8 Jun 2020 13:41:37 +0500 Subject: [PATCH 10/26] Add fields & bulk import gem --- Gemfile | 1 + Gemfile.lock | 1 + app/jobs/verify_emails_job.rb | 2 +- ...0827_create_email_address_verifications.rb | 3 + db/structure.sql | 312 ++++++++++++++---- 5 files changed, 254 insertions(+), 65 deletions(-) diff --git a/Gemfile b/Gemfile index 0804ca4db..f7556e166 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ gem 'uglifier' gem 'figaro', '1.1.1' # model related +gem 'activerecord-import' gem 'paper_trail', '~> 10.3' gem 'pg', '1.2.2' # 1.8 is for Rails < 5.0 diff --git a/Gemfile.lock b/Gemfile.lock index dde0effa4..0f3b2d355 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -494,6 +494,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import airbrake autodoc bootsnap (>= 1.1.0) diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index 4aedddb94..20c172770 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -27,7 +27,7 @@ class SendEInvoiceJob < Que::Job def log_success(verification) email = verification.try(:email) || verification - message = "Email address #{email} verified successfully" + message = "Email address #{email} verification done" logger.info message end diff --git a/db/migrate/20200605100827_create_email_address_verifications.rb b/db/migrate/20200605100827_create_email_address_verifications.rb index 39f59b51e..7f618b3a7 100644 --- a/db/migrate/20200605100827_create_email_address_verifications.rb +++ b/db/migrate/20200605100827_create_email_address_verifications.rb @@ -3,8 +3,11 @@ class CreateEmailAddressVerifications < ActiveRecord::Migration[6.0] create_table :email_address_verifications do |t| t.string :email, null: false t.datetime :verified_at + t.boolean :success, null: false, default: false + t.string :domain, null: false end add_index :email_address_verifications, :email, unique: true + add_index :email_address_verifications, :domain end end diff --git a/db/structure.sql b/db/structure.sql index 239370878..e6e736328 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -823,7 +823,9 @@ ALTER SEQUENCE public.domains_id_seq OWNED BY public.domains.id; CREATE TABLE public.email_address_verifications ( id bigint NOT NULL, email character varying NOT NULL, - verified_at timestamp without time zone + verified_at timestamp without time zone, + success boolean DEFAULT false NOT NULL, + domain character varying NOT NULL ); @@ -3613,434 +3615,616 @@ CREATE INDEX index_domains_on_registrar_id ON public.domains USING btree (regist -- --- Name: index_domains_on_statuses; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_domains_on_statuses; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_domains_on_statuses ON public.domains USING gin (statuses); -- --- Name: index_epp_sessions_on_updated_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_email_address_verifications_on_domain; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_email_address_verifications_on_domain ON public.email_address_verifications USING btree (domain); + + +-- +-- Name: index_email_address_verifications_on_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_email_address_verifications_on_email ON public.email_address_verifications USING btree (email); + + +-- +-- Name: index_email_addresses_validations_on_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_email_addresses_validations_on_email ON public.email_addresses_validations USING btree (email); + + +-- +-- Name: index_epp_sessions_on_updated_at; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_epp_sessions_on_updated_at ON public.epp_sessions USING btree (updated_at); -- --- Name: index_invoice_items_on_invoice_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_invoice_items_on_invoice_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_invoice_items_on_invoice_id ON public.invoice_items USING btree (invoice_id); -- --- Name: index_invoices_on_buyer_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_invoices_on_buyer_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_invoices_on_buyer_id ON public.invoices USING btree (buyer_id); -- --- Name: index_legal_documents_on_checksum; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_legal_documents_on_checksum; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_legal_documents_on_checksum ON public.legal_documents USING btree (checksum); -- --- Name: index_legal_documents_on_documentable_type_and_documentable_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_legal_documents_on_documentable_type_and_documentable_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_legal_documents_on_documentable_type_and_documentable_id ON public.legal_documents USING btree (documentable_type, documentable_id); -- --- Name: index_log_account_activities_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_account_activities_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_account_activities_on_item_type_and_item_id ON public.log_account_activities USING btree (item_type, item_id); -- --- Name: index_log_account_activities_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_account_activities_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_account_activities_on_whodunnit ON public.log_account_activities USING btree (whodunnit); -- --- Name: index_log_accounts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_accounts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_accounts_on_item_type_and_item_id ON public.log_accounts USING btree (item_type, item_id); -- --- Name: index_log_accounts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_accounts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_accounts_on_whodunnit ON public.log_accounts USING btree (whodunnit); -- --- Name: index_log_bank_statements_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_statements_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_statements_on_item_type_and_item_id ON public.log_bank_statements USING btree (item_type, item_id); -- --- Name: index_log_bank_statements_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_statements_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_statements_on_whodunnit ON public.log_bank_statements USING btree (whodunnit); -- --- Name: index_log_bank_transactions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_transactions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_transactions_on_item_type_and_item_id ON public.log_bank_transactions USING btree (item_type, item_id); -- --- Name: index_log_bank_transactions_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_bank_transactions_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_bank_transactions_on_whodunnit ON public.log_bank_transactions USING btree (whodunnit); -- --- Name: index_log_blocked_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_blocked_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_blocked_domains_on_item_type_and_item_id ON public.log_blocked_domains USING btree (item_type, item_id); -- --- Name: index_log_blocked_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_blocked_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_blocked_domains_on_whodunnit ON public.log_blocked_domains USING btree (whodunnit); -- --- Name: index_log_certificates_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_certificates_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_certificates_on_item_type_and_item_id ON public.log_certificates USING btree (item_type, item_id); -- --- Name: index_log_certificates_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_certificates_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_certificates_on_whodunnit ON public.log_certificates USING btree (whodunnit); -- --- Name: index_log_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_contacts_on_item_type_and_item_id ON public.log_contacts USING btree (item_type, item_id); -- --- Name: index_log_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_contacts_on_whodunnit ON public.log_contacts USING btree (whodunnit); -- --- Name: index_log_dnskeys_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_dnskeys_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_dnskeys_on_item_type_and_item_id ON public.log_dnskeys USING btree (item_type, item_id); -- --- Name: index_log_dnskeys_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_dnskeys_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_dnskeys_on_whodunnit ON public.log_dnskeys USING btree (whodunnit); -- --- Name: index_log_domain_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domain_contacts_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domain_contacts_on_item_type_and_item_id ON public.log_domain_contacts USING btree (item_type, item_id); -- --- Name: index_log_domain_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domain_contacts_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domain_contacts_on_whodunnit ON public.log_domain_contacts USING btree (whodunnit); -- --- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domains_on_item_type_and_item_id ON public.log_domains USING btree (item_type, item_id); -- --- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_domains_on_whodunnit ON public.log_domains USING btree (whodunnit); -- --- Name: index_log_invoice_items_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoice_items_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoice_items_on_item_type_and_item_id ON public.log_invoice_items USING btree (item_type, item_id); -- --- Name: index_log_invoice_items_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoice_items_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoice_items_on_whodunnit ON public.log_invoice_items USING btree (whodunnit); -- --- Name: index_log_invoices_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoices_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoices_on_item_type_and_item_id ON public.log_invoices USING btree (item_type, item_id); -- --- Name: index_log_invoices_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_invoices_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_invoices_on_whodunnit ON public.log_invoices USING btree (whodunnit); -- --- Name: index_log_nameservers_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_nameservers_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_nameservers_on_item_type_and_item_id ON public.log_nameservers USING btree (item_type, item_id); -- --- Name: index_log_nameservers_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_nameservers_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_nameservers_on_whodunnit ON public.log_nameservers USING btree (whodunnit); -- --- Name: index_log_notifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_notifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_notifications_on_item_type_and_item_id ON public.log_notifications USING btree (item_type, item_id); -- --- Name: index_log_notifications_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_notifications_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_notifications_on_whodunnit ON public.log_notifications USING btree (whodunnit); -- --- Name: index_log_registrant_verifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrant_verifications_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrant_verifications_on_item_type_and_item_id ON public.log_registrant_verifications USING btree (item_type, item_id); -- --- Name: index_log_registrant_verifications_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrant_verifications_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrant_verifications_on_whodunnit ON public.log_registrant_verifications USING btree (whodunnit); -- --- Name: index_log_registrars_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrars_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrars_on_item_type_and_item_id ON public.log_registrars USING btree (item_type, item_id); -- --- Name: index_log_registrars_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_registrars_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_registrars_on_whodunnit ON public.log_registrars USING btree (whodunnit); -- --- Name: index_log_reserved_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_reserved_domains_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_reserved_domains_on_item_type_and_item_id ON public.log_reserved_domains USING btree (item_type, item_id); -- --- Name: index_log_reserved_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_reserved_domains_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_reserved_domains_on_whodunnit ON public.log_reserved_domains USING btree (whodunnit); -- --- Name: index_log_settings_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_settings_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_settings_on_item_type_and_item_id ON public.log_settings USING btree (item_type, item_id); -- --- Name: index_log_settings_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_settings_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_settings_on_whodunnit ON public.log_settings USING btree (whodunnit); -- --- Name: index_log_users_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_users_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_users_on_item_type_and_item_id ON public.log_users USING btree (item_type, item_id); -- --- Name: index_log_users_on_whodunnit; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_log_users_on_whodunnit; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_log_users_on_whodunnit ON public.log_users USING btree (whodunnit); -- --- Name: index_nameservers_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_nameservers_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_nameservers_on_domain_id ON public.nameservers USING btree (domain_id); -- --- Name: index_notifications_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_notifications_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_notifications_on_registrar_id ON public.notifications USING btree (registrar_id); -- --- Name: index_payment_orders_on_invoice_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_payment_orders_on_invoice_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_payment_orders_on_invoice_id ON public.payment_orders USING btree (invoice_id); -- --- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_prices_on_zone_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_prices_on_zone_id ON public.prices USING btree (zone_id); -- --- Name: index_registrant_verifications_on_created_at; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_registrant_verifications_on_created_at; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_registrant_verifications_on_created_at ON public.registrant_verifications USING btree (created_at); -- --- Name: index_registrant_verifications_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_registrant_verifications_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_registrant_verifications_on_domain_id ON public.registrant_verifications USING btree (domain_id); -- --- Name: index_settings_on_thing_type_and_thing_id_and_var; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_settings_on_thing_type_and_thing_id_and_var; Type: INDEX; Schema: public; Owner: - -- CREATE UNIQUE INDEX index_settings_on_thing_type_and_thing_id_and_var ON public.settings USING btree (thing_type, thing_id, var); -- --- Name: index_users_on_identity_code; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_users_on_identity_code; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_users_on_identity_code ON public.users USING btree (identity_code); -- --- Name: index_users_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_users_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_users_on_registrar_id ON public.users USING btree (registrar_id); -- --- Name: index_versions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_versions_on_item_type_and_item_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_versions_on_item_type_and_item_id ON public.versions USING btree (item_type, item_id); -- --- Name: index_whois_records_on_domain_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_whois_records_on_domain_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_whois_records_on_domain_id ON public.whois_records USING btree (domain_id); -- --- Name: index_whois_records_on_registrar_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: index_whois_records_on_registrar_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX index_whois_records_on_registrar_id ON public.whois_records USING btree (registrar_id); -- --- Name: log_contacts_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_contacts_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_contacts_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_id'::text))::integer)); -- --- Name: log_dnskeys_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_dnskeys_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_dnskeys_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_domain_id'::text))::integer)); -- --- Name: log_domains_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_domains_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_domains_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_id'::text))::integer)); -- --- Name: log_nameservers_object_legacy_id; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: log_nameservers_object_legacy_id; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX log_nameservers_object_legacy_id ON public.log_contacts USING btree ((((object ->> 'legacy_domain_id'::text))::integer)); -- --- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: unique_data_migrations; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX unique_data_migrations ON public.data_migrations USING btree (version); + + +-- +-- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: - -- CREATE UNIQUE INDEX unique_schema_migrations ON public.schema_migrations USING btree (version); -- --- Name: contacts_registrar_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: account_activities process_account_activity_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_account_activity_audit AFTER INSERT OR DELETE OR UPDATE ON public.account_activities FOR EACH ROW EXECUTE PROCEDURE public.process_account_activity_audit(); + + +-- +-- Name: accounts process_account_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_account_audit AFTER INSERT OR DELETE OR UPDATE ON public.accounts FOR EACH ROW EXECUTE PROCEDURE public.process_account_audit(); + + +-- +-- Name: actions process_action_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_action_audit AFTER INSERT OR DELETE OR UPDATE ON public.actions FOR EACH ROW EXECUTE PROCEDURE public.process_action_audit(); + + +-- +-- Name: bank_statements process_bank_statement_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_bank_statement_audit AFTER INSERT OR DELETE OR UPDATE ON public.bank_statements FOR EACH ROW EXECUTE PROCEDURE public.process_bank_statement_audit(); + + +-- +-- Name: bank_transactions process_bank_transaction_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_bank_transaction_audit AFTER INSERT OR DELETE OR UPDATE ON public.bank_transactions FOR EACH ROW EXECUTE PROCEDURE public.process_bank_transaction_audit(); + + +-- +-- Name: blocked_domains process_blocked_domain_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_blocked_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.blocked_domains FOR EACH ROW EXECUTE PROCEDURE public.process_blocked_domain_audit(); + + +-- +-- Name: certificates process_certificate_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_certificate_audit AFTER INSERT OR DELETE OR UPDATE ON public.certificates FOR EACH ROW EXECUTE PROCEDURE public.process_certificate_audit(); + + +-- +-- Name: contacts process_contact_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_contact_audit AFTER INSERT OR DELETE OR UPDATE ON public.contacts FOR EACH ROW EXECUTE PROCEDURE public.process_contact_audit(); + + +-- +-- Name: dnskeys process_dnskey_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_dnskey_audit AFTER INSERT OR DELETE OR UPDATE ON public.dnskeys FOR EACH ROW EXECUTE PROCEDURE public.process_dnskey_audit(); + + +-- +-- Name: domains process_domain_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.domains FOR EACH ROW EXECUTE PROCEDURE public.process_domain_audit(); + + +-- +-- Name: domain_contacts process_domain_contact_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_domain_contact_audit AFTER INSERT OR DELETE OR UPDATE ON public.domain_contacts FOR EACH ROW EXECUTE PROCEDURE public.process_domain_contact_audit(); + + +-- +-- Name: invoices process_invoice_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_invoice_audit AFTER INSERT OR DELETE OR UPDATE ON public.invoices FOR EACH ROW EXECUTE PROCEDURE public.process_invoice_audit(); + + +-- +-- Name: invoice_items process_invoice_item_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_invoice_item_audit AFTER INSERT OR DELETE OR UPDATE ON public.invoice_items FOR EACH ROW EXECUTE PROCEDURE public.process_invoice_item_audit(); + + +-- +-- Name: nameservers process_nameserver_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_nameserver_audit AFTER INSERT OR DELETE OR UPDATE ON public.nameservers FOR EACH ROW EXECUTE PROCEDURE public.process_nameserver_audit(); + + +-- +-- Name: notifications process_notification_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_notification_audit AFTER INSERT OR DELETE OR UPDATE ON public.notifications FOR EACH ROW EXECUTE PROCEDURE public.process_notification_audit(); + + +-- +-- Name: payment_orders process_payment_order_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_payment_order_audit AFTER INSERT OR DELETE OR UPDATE ON public.payment_orders FOR EACH ROW EXECUTE PROCEDURE public.process_payment_order_audit(); + + +-- +-- Name: registrant_verifications process_registrant_verification_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_registrant_verification_audit AFTER INSERT OR DELETE OR UPDATE ON public.registrant_verifications FOR EACH ROW EXECUTE PROCEDURE public.process_registrant_verification_audit(); + + +-- +-- Name: registrars process_registrar_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_registrar_audit AFTER INSERT OR DELETE OR UPDATE ON public.registrars FOR EACH ROW EXECUTE PROCEDURE public.process_registrar_audit(); + + +-- +-- Name: reserved_domains process_reserved_domain_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_reserved_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.reserved_domains FOR EACH ROW EXECUTE PROCEDURE public.process_reserved_domain_audit(); + + +-- +-- Name: settings process_setting_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_setting_audit AFTER INSERT OR DELETE OR UPDATE ON public.settings FOR EACH ROW EXECUTE PROCEDURE public.process_setting_audit(); + + +-- +-- Name: users process_user_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_user_audit AFTER INSERT OR DELETE OR UPDATE ON public.users FOR EACH ROW EXECUTE PROCEDURE public.process_user_audit(); + + +-- +-- Name: white_ips process_white_ip_audit; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER process_white_ip_audit AFTER INSERT OR DELETE OR UPDATE ON public.white_ips FOR EACH ROW EXECUTE PROCEDURE public.process_white_ip_audit(); + + +-- +-- Name: contacts contacts_registrar_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.contacts @@ -4048,7 +4232,7 @@ ALTER TABLE ONLY public.contacts -- --- Name: domain_contacts_contact_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: domain_contacts domain_contacts_contact_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY public.domain_contacts From ee7d69c1ce8454ee321602a25a48b3fe9c94bdb8 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 8 Jun 2020 14:12:39 +0500 Subject: [PATCH 11/26] Add data migration to fill EmailAddressVerifications --- app/models/concerns/email_checkable.rb | 8 +- app/models/email_address_verification.rb | 7 +- ...20200608084321_fill_email_verifications.rb | 19 +++ db/structure.sql | 155 ------------------ 4 files changed, 29 insertions(+), 160 deletions(-) create mode 100644 db/data/20200608084321_fill_email_verifications.rb diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_checkable.rb index 3edb9614d..5c82a0f9a 100644 --- a/app/models/concerns/email_checkable.rb +++ b/app/models/concerns/email_checkable.rb @@ -3,12 +3,14 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_or_create_by(email: self.email) + EmailAddressVerification.find_or_create_by(email: self.email, + domain: Mail::Address.new(self.email).domain) end def billing_email_verification if self.attribute_names.include?('billing_email') - EmailAddressVerification.find_or_create_by(email: self.billing_email) + EmailAddressVerification.find_or_create_by(email: self.billing_email, + domain: Mail::Address.new(self.email).domain) else nil end @@ -23,8 +25,6 @@ module Concerns end def correct_billing_email_format - return if self[:billing_email].blank? - verify_email_mx_smtp(field: :billing_email, email: billing_email) end end diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 8afc74a78..edcdbaa35 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -10,6 +10,11 @@ class EmailAddressVerification < ApplicationRecord def verify validation_request = Truemail.validate(email) - update(verified_at: Time.zone.now) if validation_request.result.success + if validation_request.result.success + update(verified_at: Time.zone.now, + success: true) + end + + validation_request.result.success end end diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb new file mode 100644 index 000000000..f12381b3c --- /dev/null +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -0,0 +1,19 @@ +class FillEmailVerifications < ActiveRecord::Migration[6.0] + def up + registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?) + registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?) + contact_emails = Contact.pluck(:email).uniq.reject(&:blank?) + + emails = (contact_emails || registrar_emails || registrar_billing_emails).uniq + + result = emails.map do |email| + { email: email, domain: Mail::Address.new(email).domain || 'not_found' } + end + + EmailAddressVerification.import result, batch_size: 500 + end + + def down + EmailAddressVerification.delete_all + end +end diff --git a/db/structure.sql b/db/structure.sql index e6e736328..c5ad902b7 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4068,161 +4068,6 @@ CREATE UNIQUE INDEX unique_data_migrations ON public.data_migrations USING btree CREATE UNIQUE INDEX unique_schema_migrations ON public.schema_migrations USING btree (version); - --- --- Name: account_activities process_account_activity_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_account_activity_audit AFTER INSERT OR DELETE OR UPDATE ON public.account_activities FOR EACH ROW EXECUTE PROCEDURE public.process_account_activity_audit(); - - --- --- Name: accounts process_account_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_account_audit AFTER INSERT OR DELETE OR UPDATE ON public.accounts FOR EACH ROW EXECUTE PROCEDURE public.process_account_audit(); - - --- --- Name: actions process_action_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_action_audit AFTER INSERT OR DELETE OR UPDATE ON public.actions FOR EACH ROW EXECUTE PROCEDURE public.process_action_audit(); - - --- --- Name: bank_statements process_bank_statement_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_bank_statement_audit AFTER INSERT OR DELETE OR UPDATE ON public.bank_statements FOR EACH ROW EXECUTE PROCEDURE public.process_bank_statement_audit(); - - --- --- Name: bank_transactions process_bank_transaction_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_bank_transaction_audit AFTER INSERT OR DELETE OR UPDATE ON public.bank_transactions FOR EACH ROW EXECUTE PROCEDURE public.process_bank_transaction_audit(); - - --- --- Name: blocked_domains process_blocked_domain_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_blocked_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.blocked_domains FOR EACH ROW EXECUTE PROCEDURE public.process_blocked_domain_audit(); - - --- --- Name: certificates process_certificate_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_certificate_audit AFTER INSERT OR DELETE OR UPDATE ON public.certificates FOR EACH ROW EXECUTE PROCEDURE public.process_certificate_audit(); - - --- --- Name: contacts process_contact_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_contact_audit AFTER INSERT OR DELETE OR UPDATE ON public.contacts FOR EACH ROW EXECUTE PROCEDURE public.process_contact_audit(); - - --- --- Name: dnskeys process_dnskey_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_dnskey_audit AFTER INSERT OR DELETE OR UPDATE ON public.dnskeys FOR EACH ROW EXECUTE PROCEDURE public.process_dnskey_audit(); - - --- --- Name: domains process_domain_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.domains FOR EACH ROW EXECUTE PROCEDURE public.process_domain_audit(); - - --- --- Name: domain_contacts process_domain_contact_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_domain_contact_audit AFTER INSERT OR DELETE OR UPDATE ON public.domain_contacts FOR EACH ROW EXECUTE PROCEDURE public.process_domain_contact_audit(); - - --- --- Name: invoices process_invoice_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_invoice_audit AFTER INSERT OR DELETE OR UPDATE ON public.invoices FOR EACH ROW EXECUTE PROCEDURE public.process_invoice_audit(); - - --- --- Name: invoice_items process_invoice_item_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_invoice_item_audit AFTER INSERT OR DELETE OR UPDATE ON public.invoice_items FOR EACH ROW EXECUTE PROCEDURE public.process_invoice_item_audit(); - - --- --- Name: nameservers process_nameserver_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_nameserver_audit AFTER INSERT OR DELETE OR UPDATE ON public.nameservers FOR EACH ROW EXECUTE PROCEDURE public.process_nameserver_audit(); - - --- --- Name: notifications process_notification_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_notification_audit AFTER INSERT OR DELETE OR UPDATE ON public.notifications FOR EACH ROW EXECUTE PROCEDURE public.process_notification_audit(); - - --- --- Name: payment_orders process_payment_order_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_payment_order_audit AFTER INSERT OR DELETE OR UPDATE ON public.payment_orders FOR EACH ROW EXECUTE PROCEDURE public.process_payment_order_audit(); - - --- --- Name: registrant_verifications process_registrant_verification_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_registrant_verification_audit AFTER INSERT OR DELETE OR UPDATE ON public.registrant_verifications FOR EACH ROW EXECUTE PROCEDURE public.process_registrant_verification_audit(); - - --- --- Name: registrars process_registrar_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_registrar_audit AFTER INSERT OR DELETE OR UPDATE ON public.registrars FOR EACH ROW EXECUTE PROCEDURE public.process_registrar_audit(); - - --- --- Name: reserved_domains process_reserved_domain_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_reserved_domain_audit AFTER INSERT OR DELETE OR UPDATE ON public.reserved_domains FOR EACH ROW EXECUTE PROCEDURE public.process_reserved_domain_audit(); - - --- --- Name: settings process_setting_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_setting_audit AFTER INSERT OR DELETE OR UPDATE ON public.settings FOR EACH ROW EXECUTE PROCEDURE public.process_setting_audit(); - - --- --- Name: users process_user_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_user_audit AFTER INSERT OR DELETE OR UPDATE ON public.users FOR EACH ROW EXECUTE PROCEDURE public.process_user_audit(); - - --- --- Name: white_ips process_white_ip_audit; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER process_white_ip_audit AFTER INSERT OR DELETE OR UPDATE ON public.white_ips FOR EACH ROW EXECUTE PROCEDURE public.process_white_ip_audit(); - - -- -- Name: contacts contacts_registrar_id_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- From a13725faa7c9bad27269b156ff992bfd76716efe Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 8 Jun 2020 16:09:52 +0500 Subject: [PATCH 12/26] Add email verification rake task --- app/jobs/verify_emails_job.rb | 9 +++++---- app/models/email_address_verification.rb | 22 +++++++++++++++++++--- lib/tasks/verify_email.rake | 11 +++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 lib/tasks/verify_email.rake diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index 20c172770..d7d512961 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -1,8 +1,7 @@ -class SendEInvoiceJob < Que::Job +class VerifyEmailsJob < Que::Job def run(verification_id) - email_address_verification = run_condition(EmailAddressVerification - .find(run_condition(verification_id))) + email_address_verification = run_condition(EmailAddressVerification.find(verification_id)) ActiveRecord::Base.transaction do email_address_verification.verify @@ -17,12 +16,14 @@ class SendEInvoiceJob < Que::Job private def run_condition(email_address_verification) + destroy unless email_address_verification destroy if email_address_verification.recently_verified? + email_address_verification end def logger - Rails.logger + Rails.logger = Logger.new(STDOUT) end def log_success(verification) diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index edcdbaa35..57b75fcb4 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -1,14 +1,30 @@ class EmailAddressVerification < ApplicationRecord - RECENTLY_VERIFIED_PERIOD = 1.month + scope :not_verified_recently, -> { + where('verified_at IS NULL or verified_at < ?', verification_period) + } + + scope :verified_recently, -> { + where('verified_at IS NOT NULL and verified_at >= ?', verification_period) + } + def recently_verified? verified_at.present? && - verified_at > Time.zone.now - RECENTLY_VERIFIED_PERIOD + verified_at > verification_period + end + + def verification_period + self.class.verification_period + end + + def self.verification_period + Time.zone.now - RECENTLY_VERIFIED_PERIOD end def verify - validation_request = Truemail.validate(email) + media = success ? :mx : :smtp + validation_request = Truemail.validate(email, with: media) if validation_request.result.success update(verified_at: Time.zone.now, diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake new file mode 100644 index 000000000..da94e7edb --- /dev/null +++ b/lib/tasks/verify_email.rake @@ -0,0 +1,11 @@ +namespace :verify_email do + desc 'Stars verifying email jobs' + task all_domains: :environment do + verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) + + verifications_by_domain.each do |_domain, verifications| + ver = verifications[0] # Only first email to not to clog the SMTP servers + VerifyEmailsJob.enqueue(ver.id) + end + end +end From 3d444c3a04bad57de0e37bf2aa94ade90a2aa6d3 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 8 Jun 2020 16:16:06 +0500 Subject: [PATCH 13/26] Fix CC issues --- app/jobs/verify_emails_job.rb | 1 - app/models/concerns/email_checkable.rb | 20 +++++++++++-------- app/models/email_address_verification.rb | 7 ++++--- ...20200608084321_fill_email_verifications.rb | 8 +++++++- lib/tasks/verify_email.rake | 5 ++++- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index d7d512961..4da854870 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -1,5 +1,4 @@ class VerifyEmailsJob < Que::Job - def run(verification_id) email_address_verification = run_condition(EmailAddressVerification.find(verification_id)) diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_checkable.rb index 5c82a0f9a..ba5784336 100644 --- a/app/models/concerns/email_checkable.rb +++ b/app/models/concerns/email_checkable.rb @@ -3,17 +3,21 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_or_create_by(email: self.email, - domain: Mail::Address.new(self.email).domain) + EmailAddressVerification.find_or_create_by(email: email, + domain: domain(email)) end def billing_email_verification - if self.attribute_names.include?('billing_email') - EmailAddressVerification.find_or_create_by(email: self.billing_email, - domain: Mail::Address.new(self.email).domain) - else - nil - end + return unless attribute_names.include?('billing_email') + + EmailAddressVerification.find_or_create_by(email: billing_email, + domain: domain(email)) + end + + def domain(email) + Mail::Address.new(email).domain || 'not_found' + rescue Mail::Field::IncompleteParseError + 'not_found' end def verify_email_mx_smtp(field:, email:) diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 57b75fcb4..bed0cfae3 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -1,11 +1,11 @@ class EmailAddressVerification < ApplicationRecord RECENTLY_VERIFIED_PERIOD = 1.month - scope :not_verified_recently, -> { + scope :not_verified_recently, lambda { where('verified_at IS NULL or verified_at < ?', verification_period) } - scope :verified_recently, -> { + scope :verified_recently, lambda { where('verified_at IS NOT NULL and verified_at >= ?', verification_period) } @@ -23,7 +23,8 @@ class EmailAddressVerification < ApplicationRecord end def verify - media = success ? :mx : :smtp + # media = success ? :mx : :smtp + media = :mx validation_request = Truemail.validate(email, with: media) if validation_request.result.success diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb index f12381b3c..7d3da34ce 100644 --- a/db/data/20200608084321_fill_email_verifications.rb +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -7,7 +7,7 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] emails = (contact_emails || registrar_emails || registrar_billing_emails).uniq result = emails.map do |email| - { email: email, domain: Mail::Address.new(email).domain || 'not_found' } + { email: email, domain: domain(email) } end EmailAddressVerification.import result, batch_size: 500 @@ -16,4 +16,10 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] def down EmailAddressVerification.delete_all end + + def domain(email) + Mail::Address.new(email).domain || 'not_found' + rescue Mail::Field::IncompleteParseError + 'not_found' + end end diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index da94e7edb..c97a8e39b 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -3,9 +3,12 @@ namespace :verify_email do task all_domains: :environment do verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) - verifications_by_domain.each do |_domain, verifications| + verifications_by_domain.each do |domain, verifications| + next if domain == 'not_found' + ver = verifications[0] # Only first email to not to clog the SMTP servers VerifyEmailsJob.enqueue(ver.id) + # VerifyEmailsJob.run(ver.id) end end end From 48036c660d83a1f78eeae8cf9929973cd5d3814e Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 9 Jun 2020 14:05:52 +0500 Subject: [PATCH 14/26] Add email & nonverified color to contact show --- .../concerns/{email_checkable.rb => email_verifable.rb} | 2 +- app/models/contact.rb | 2 +- app/models/registrar.rb | 2 +- app/views/admin/contacts/index.haml | 6 +++++- app/views/admin/contacts/partials/_general.haml | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) rename app/models/concerns/{email_checkable.rb => email_verifable.rb} (97%) diff --git a/app/models/concerns/email_checkable.rb b/app/models/concerns/email_verifable.rb similarity index 97% rename from app/models/concerns/email_checkable.rb rename to app/models/concerns/email_verifable.rb index ba5784336..de8041c32 100644 --- a/app/models/concerns/email_checkable.rb +++ b/app/models/concerns/email_verifable.rb @@ -1,5 +1,5 @@ module Concerns - module EmailCheckable + module EmailVerifable extend ActiveSupport::Concern def email_verification diff --git a/app/models/contact.rb b/app/models/contact.rb index 4b60e85be..3fec501c8 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -7,7 +7,7 @@ class Contact < ApplicationRecord include Concerns::Contact::Transferable include Concerns::Contact::Identical include Concerns::Contact::Disclosable - include Concerns::EmailCheckable + include Concerns::EmailVerifable belongs_to :original, class_name: self.name belongs_to :registrar, required: true diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 6e6d41b77..470d768b7 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -1,7 +1,7 @@ class Registrar < ApplicationRecord include Versions # version/registrar_version.rb include Concerns::Registrar::BookKeeping - include Concerns::EmailCheckable + include Concerns::EmailVerifable include Concerns::Registrar::LegalDoc has_many :domains, dependent: :restrict_with_error diff --git a/app/views/admin/contacts/index.haml b/app/views/admin/contacts/index.haml index cc80ac744..7ba2220a8 100644 --- a/app/views/admin/contacts/index.haml +++ b/app/views/admin/contacts/index.haml @@ -85,7 +85,9 @@ %th{class: 'col-xs-2'} = sort_link(@q, 'ident', t(:ident)) %th{class: 'col-xs-2'} - = sort_link(@q, 'email', t(:created_at)) + = sort_link(@q, 'email', t(:email)) + %th{class: 'col-xs-2'} + = sort_link(@q, 'created_at', t(:created_at)) %th{class: 'col-xs-2'} = sort_link(@q, 'registrar_name', t(:registrar_name)) %tbody @@ -94,6 +96,8 @@ %td= link_to(contact, admin_contact_path(contact)) %td= contact.code %td= ident_for(contact) + %td{class: ('text-danger' unless contact.email_verification.success)} + = contact.email %td= l(contact.created_at, format: :short) %td - if contact.registrar diff --git a/app/views/admin/contacts/partials/_general.haml b/app/views/admin/contacts/partials/_general.haml index 029f89509..7245ca218 100644 --- a/app/views/admin/contacts/partials/_general.haml +++ b/app/views/admin/contacts/partials/_general.haml @@ -17,7 +17,8 @@ %dd= ident_for(@contact) %dt= t(:email) - %dd= @contact.email + %dd{class: ('text-danger' unless @contact.email_verification.success)} + = @contact.email %dt= t(:phone) %dd= @contact.phone From af7e0f266d8d37e8ef5fa0367e7e43a38513fb33 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 9 Jun 2020 14:57:55 +0500 Subject: [PATCH 15/26] Add non-verified email filreting in contact index --- app/controllers/admin/contacts_controller.rb | 6 ++++-- app/models/contact.rb | 9 +++++++++ app/views/admin/contacts/index.haml | 4 ++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/admin/contacts_controller.rb b/app/controllers/admin/contacts_controller.rb index 4eea4faad..d48d5c09b 100644 --- a/app/controllers/admin/contacts_controller.rb +++ b/app/controllers/admin/contacts_controller.rb @@ -13,10 +13,12 @@ module Admin search_params[:registrant_domains_id_not_null] = 1 end - contacts = Contact.includes(:registrar).joins(:registrar).select('contacts.*, registrars.name') + contacts = Contact.includes(:registrar).joins(:registrar) + .includes(:email_address_verification) + .select('contacts.*, registrars.name') contacts = contacts.filter_by_states(params[:statuses_contains].join(',')) if params[:statuses_contains] contacts = contacts.where("ident_country_code is null or ident_country_code=''") if params[:only_no_country_code].eql?('1') - + contacts = contacts.email_not_verified if params[:email_not_verified].eql?('1') normalize_search_parameters do @q = contacts.search(search_params) diff --git a/app/models/contact.rb b/app/models/contact.rb index 3fec501c8..556907ba4 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -16,6 +16,10 @@ class Contact < ApplicationRecord has_many :legal_documents, as: :documentable has_many :registrant_domains, class_name: 'Domain', foreign_key: 'registrant_id' has_many :actions, dependent: :destroy + belongs_to :email_address_verification, class_name: 'EmailAddressVerification', + primary_key: 'email', + foreign_key: 'email', + optional: true attr_accessor :legal_document_id alias_attribute :kind, :ident_type @@ -23,6 +27,11 @@ class Contact < ApplicationRecord accepts_nested_attributes_for :legal_documents + scope :email_not_verified, lambda { + joins(:email_address_verification) + .where('verified_at IS NULL OR verified_at <= ?', EmailAddressVerification.verification_period) + } + validates :name, :email, presence: true validates :street, :city, :zip, :country_code, presence: true, if: lambda { self.class.address_processing? diff --git a/app/views/admin/contacts/index.haml b/app/views/admin/contacts/index.haml index 7ba2220a8..bf3665664 100644 --- a/app/views/admin/contacts/index.haml +++ b/app/views/admin/contacts/index.haml @@ -63,6 +63,10 @@ .form-group = label_tag :only_no_country_code, "Ident CC missing" = check_box_tag :only_no_country_code, '1',params[:only_no_country_code].eql?('1'), style: 'width:auto;height:auto;float:right' + .col-md-3 + .form-group + = label_tag :email_not_verified, "Email not verified" + = check_box_tag :email_not_verified, '1',params[:email_not_verified].eql?('1'), style: 'width:auto;height:auto;float:right' .row .col-md-3{style: 'padding-top: 25px;float:right;'} From e95996fa12059646cb76fc68e3b28ba5d71cb703 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 9 Jun 2020 15:51:51 +0500 Subject: [PATCH 16/26] Add nonverified color email to registrar index --- app/models/email_address_verification.rb | 3 +++ app/models/registrar.rb | 8 ++++++++ app/views/admin/registrars/index.html.erb | 13 +++++++++++++ config/locales/en.yml | 1 + config/locales/et.yml | 1 + db/data/20200608084321_fill_email_verifications.rb | 3 +-- lib/tasks/verify_email.rake | 5 ++--- 7 files changed, 29 insertions(+), 5 deletions(-) diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index bed0cfae3..412ce7f83 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -30,6 +30,9 @@ class EmailAddressVerification < ApplicationRecord if validation_request.result.success update(verified_at: Time.zone.now, success: true) + else + update(verified_at: nil, + success: false) end validation_request.result.success diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 470d768b7..3fec53f21 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -13,6 +13,14 @@ class Registrar < ApplicationRecord has_many :nameservers, through: :domains has_many :whois_records has_many :white_ips, dependent: :destroy + belongs_to :email_address_verification, class_name: 'EmailAddressVerification', + primary_key: 'email', + foreign_key: 'email', + optional: true + belongs_to :billing_email_address_verification, class_name: 'EmailAddressVerification', + primary_key: 'email', + foreign_key: 'billing_email', + optional: true delegate :balance, to: :cash_account, allow_nil: true diff --git a/app/views/admin/registrars/index.html.erb b/app/views/admin/registrars/index.html.erb index a66816568..4b3fbd98a 100644 --- a/app/views/admin/registrars/index.html.erb +++ b/app/views/admin/registrars/index.html.erb @@ -28,6 +28,9 @@ <%= t(:test_registrar) %> + + <%= t(:emails) %> + @@ -45,6 +48,16 @@ <%= "#{x.test_registrar}" %> + + > + <%= "#{x.email}" %> + + <% if x[:billing_email].present? %> + > + <%= "#{x[:billing_email]}" %> + + <% end %> + <% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index a825b1dc0..27299072e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -636,6 +636,7 @@ en: edit_dispute: 'Edit dispute' optional: 'Optional' test_registrar: "Test registrar" + emails: 'Email addresses' verified_confirm: 'Verified status is for cases when current registrant is the one applying for the update. Legal document signed by the registrant is required. Are you sure this update is properly verified with the registrant?' verified: 'Verified' deleted: 'Deleted' diff --git a/config/locales/et.yml b/config/locales/et.yml index 05d32be24..9cb8aaa4a 100644 --- a/config/locales/et.yml +++ b/config/locales/et.yml @@ -5,3 +5,4 @@ et: date: # Don't forget the nil at the beginning; there's no such thing as a 0th month month_names: [~, Jaanuar, Veebruar, Märts, Aprill, Mai, Juuni, Juuli, August, September, Oktoober, November, Detsember] + emails: "Meillaadressid" diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb index 7d3da34ce..68f02b133 100644 --- a/db/data/20200608084321_fill_email_verifications.rb +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -4,12 +4,11 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?) contact_emails = Contact.pluck(:email).uniq.reject(&:blank?) - emails = (contact_emails || registrar_emails || registrar_billing_emails).uniq + emails = (contact_emails + registrar_emails + registrar_billing_emails).uniq result = emails.map do |email| { email: email, domain: domain(email) } end - EmailAddressVerification.import result, batch_size: 500 end diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index c97a8e39b..d069100f2 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -2,13 +2,12 @@ namespace :verify_email do desc 'Stars verifying email jobs' task all_domains: :environment do verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) - verifications_by_domain.each do |domain, verifications| next if domain == 'not_found' - ver = verifications[0] # Only first email to not to clog the SMTP servers + ver = verifications.sample # Verify random email to not to clog the SMTP servers VerifyEmailsJob.enqueue(ver.id) - # VerifyEmailsJob.run(ver.id) + next end end end From 62acd8892a93b45109950e7e14d3dd366a9e4deb Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 9 Jun 2020 15:56:45 +0500 Subject: [PATCH 17/26] Add nonverified email color to registrar show --- app/controllers/admin/contacts_controller.rb | 12 +++++++++--- app/models/contact.rb | 9 +++------ app/models/registrar.rb | 18 ++++++++++-------- .../admin/contacts/partials/_general.haml | 2 +- app/views/admin/registrars/index.html.erb | 4 ++-- .../admin/registrars/show/_billing.html.erb | 6 ++++-- .../admin/registrars/show/_contacts.html.erb | 6 ++++-- 7 files changed, 33 insertions(+), 24 deletions(-) diff --git a/app/controllers/admin/contacts_controller.rb b/app/controllers/admin/contacts_controller.rb index d48d5c09b..685a08c86 100644 --- a/app/controllers/admin/contacts_controller.rb +++ b/app/controllers/admin/contacts_controller.rb @@ -14,11 +14,9 @@ module Admin end contacts = Contact.includes(:registrar).joins(:registrar) - .includes(:email_address_verification) .select('contacts.*, registrars.name') contacts = contacts.filter_by_states(params[:statuses_contains].join(',')) if params[:statuses_contains] - contacts = contacts.where("ident_country_code is null or ident_country_code=''") if params[:only_no_country_code].eql?('1') - contacts = contacts.email_not_verified if params[:email_not_verified].eql?('1') + contacts = filter_by_flags(contacts) normalize_search_parameters do @q = contacts.search(search_params) @@ -28,6 +26,14 @@ module Admin @contacts = @contacts.per(params[:results_per_page]) if params[:results_per_page].to_i.positive? end + def filter_by_flags(contacts) + if params[:only_no_country_code].eql?('1') + contacts = contacts.where("ident_country_code is null or ident_country_code=''") + end + contacts = contacts.email_not_verified if params[:email_not_verified].eql?('1') + contacts + end + def search render json: Contact.search_by_query(params[:q]) end diff --git a/app/models/contact.rb b/app/models/contact.rb index 556907ba4..a24a53a72 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -16,10 +16,6 @@ class Contact < ApplicationRecord has_many :legal_documents, as: :documentable has_many :registrant_domains, class_name: 'Domain', foreign_key: 'registrant_id' has_many :actions, dependent: :destroy - belongs_to :email_address_verification, class_name: 'EmailAddressVerification', - primary_key: 'email', - foreign_key: 'email', - optional: true attr_accessor :legal_document_id alias_attribute :kind, :ident_type @@ -28,8 +24,9 @@ class Contact < ApplicationRecord accepts_nested_attributes_for :legal_documents scope :email_not_verified, lambda { - joins(:email_address_verification) - .where('verified_at IS NULL OR verified_at <= ?', EmailAddressVerification.verification_period) + joins('LEFT JOIN :email_address_verifications emv ON contacts.email = emv.email') + .where('verified_at IS NULL OR verified_at <= ?', + EmailAddressVerification.verification_period) } validates :name, :email, presence: true diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 3fec53f21..9abdfdb68 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -13,14 +13,16 @@ class Registrar < ApplicationRecord has_many :nameservers, through: :domains has_many :whois_records has_many :white_ips, dependent: :destroy - belongs_to :email_address_verification, class_name: 'EmailAddressVerification', - primary_key: 'email', - foreign_key: 'email', - optional: true - belongs_to :billing_email_address_verification, class_name: 'EmailAddressVerification', - primary_key: 'email', - foreign_key: 'billing_email', - optional: true + # belongs_to :email_address_verification, class_name: 'EmailAddressVerification', + # primary_key: 'email', + # foreign_key: 'email', + # optional: true, + # inverse_of: :registrar + # belongs_to :billing_email_address_verification, class_name: 'EmailAddressVerification', + # primary_key: 'email', + # foreign_key: 'billing_email', + # optional: true, + # inverse_of: :billing_registrar delegate :balance, to: :cash_account, allow_nil: true diff --git a/app/views/admin/contacts/partials/_general.haml b/app/views/admin/contacts/partials/_general.haml index 7245ca218..c4a151294 100644 --- a/app/views/admin/contacts/partials/_general.haml +++ b/app/views/admin/contacts/partials/_general.haml @@ -17,7 +17,7 @@ %dd= ident_for(@contact) %dt= t(:email) - %dd{class: ('text-danger' unless @contact.email_verification.success)} + %dd{class: ('text-danger' unless @contact.email_verification&.success)} = @contact.email %dt= t(:phone) diff --git a/app/views/admin/registrars/index.html.erb b/app/views/admin/registrars/index.html.erb index 4b3fbd98a..6ed864972 100644 --- a/app/views/admin/registrars/index.html.erb +++ b/app/views/admin/registrars/index.html.erb @@ -49,11 +49,11 @@ <%= "#{x.test_registrar}" %> - > + > <%= "#{x.email}" %> <% if x[:billing_email].present? %> - > + > <%= "#{x[:billing_email]}" %> <% end %> diff --git a/app/views/admin/registrars/show/_billing.html.erb b/app/views/admin/registrars/show/_billing.html.erb index da79b9074..ab731ce1d 100644 --- a/app/views/admin/registrars/show/_billing.html.erb +++ b/app/views/admin/registrars/show/_billing.html.erb @@ -15,7 +15,9 @@
<%= registrar.accounting_customer_code %>
<%= Registrar.human_attribute_name :billing_email %>
-
<%= registrar.billing_email %>
+
> + <%= registrar.billing_email %> +
<%= Registrar.human_attribute_name :reference_no %>
<%= registrar.reference_no %>
@@ -24,4 +26,4 @@
<%= registrar.iban %>
- \ No newline at end of file + diff --git a/app/views/admin/registrars/show/_contacts.html.erb b/app/views/admin/registrars/show/_contacts.html.erb index f467e6a51..b5a222dd5 100644 --- a/app/views/admin/registrars/show/_contacts.html.erb +++ b/app/views/admin/registrars/show/_contacts.html.erb @@ -15,7 +15,9 @@
<%= @registrar.phone %>
<%= Registrar.human_attribute_name :email %>
-
<%= @registrar.email %>
+
> + <%= @registrar.email %> +
- \ No newline at end of file + From 438a2e05ab949ba5751851be199b733e078c9c7d Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 10 Jun 2020 13:27:54 +0500 Subject: [PATCH 18/26] Add test for email verification job --- app/jobs/verify_emails_job.rb | 7 +++- test/fixtures/contacts.yml | 8 ++++ test/jobs/verify_emails_job_test.rb | 59 +++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 test/jobs/verify_emails_job_test.rb diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index 4da854870..a0d7234c5 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -2,6 +2,11 @@ class VerifyEmailsJob < Que::Job def run(verification_id) email_address_verification = run_condition(EmailAddressVerification.find(verification_id)) + if email_address_verification.recently_verified? + destroy + return + end + ActiveRecord::Base.transaction do email_address_verification.verify log_success(email_address_verification) @@ -22,7 +27,7 @@ class VerifyEmailsJob < Que::Job end def logger - Rails.logger = Logger.new(STDOUT) + Rails.logger end def log_success(verification) diff --git a/test/fixtures/contacts.yml b/test/fixtures/contacts.yml index ddfbfe93f..0173d56dd 100644 --- a/test/fixtures/contacts.yml +++ b/test/fixtures/contacts.yml @@ -85,3 +85,11 @@ invalid: auth_info: any registrar: bestnames uuid: bd80c0f9-26ee-49e0-a2cb-2311d931c433 + +invalid_email: + name: any + code: invalid_email + email: invalid@invalid. + auth_info: any + registrar: bestnames + uuid: fa8c4f51-a221-4628-b3c6-47995f4edea3 diff --git a/test/jobs/verify_emails_job_test.rb b/test/jobs/verify_emails_job_test.rb new file mode 100644 index 000000000..f55a474db --- /dev/null +++ b/test/jobs/verify_emails_job_test.rb @@ -0,0 +1,59 @@ +require "test_helper" + +class VerifyEmailsJobTest < ActiveSupport::TestCase + def setup + @contact = contacts(:john) + @invalid_contact = contacts(:invalid_email) + @contact_verification = @contact.email_verification + @invalid_contact_verification = @invalid_contact.email_verification + + @default_whitelist = Truemail.configure.whitelisted_domains + @default_blacklist = Truemail.configure.blacklisted_domains + Truemail.configure.whitelisted_domains = whitelisted_domains + Truemail.configure.blacklisted_domains = blacklisted_domains + end + + def teardown + Truemail.configure.whitelisted_domains = @default_whitelist + Truemail.configure.blacklisted_domains = @default_blacklist + end + + def domain(email) + Mail::Address.new(email).domain + rescue Mail::Field::IncompleteParseError + nil + end + + def whitelisted_domains + [domain(@contact.email)].reject(&:blank?) + end + + def blacklisted_domains + [domain(@invalid_contact.email)].reject(&:blank?) + end + + def test_job_checks_if_email_valid + VerifyEmailsJob.run(@contact_verification.id) + @contact_verification.reload + + assert @contact_verification.success + end + + def test_job_checks_does_not_run_if_recent + old_verified_at = Time.zone.now - 10.days + @contact_verification.update(success: true, verified_at: old_verified_at) + assert @contact_verification.recently_verified? + + VerifyEmailsJob.run(@contact_verification.id) + @contact_verification.reload + + assert_in_delta @contact_verification.verified_at.to_i, old_verified_at.to_i, 1 + end + + def test_job_checks_if_email_invalid + VerifyEmailsJob.run(@invalid_contact_verification.id) + @contact_verification.reload + + refute @contact_verification.success + end +end From f8eea0835799e06ec9fb189a3e868683a3656c1a Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 10 Jun 2020 14:14:03 +0500 Subject: [PATCH 19/26] Move emails & domains to citext, CC fixes --- app/jobs/verify_emails_job.rb | 5 +---- app/models/concerns/email_verifable.rb | 8 ++++---- config/schedule.rb | 4 ++++ ...change_email_verification_fields_to_citext.rb | 13 +++++++++++++ db/structure.sql | 16 ++++++++++++---- 5 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 db/migrate/20200610090110_change_email_verification_fields_to_citext.rb diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index a0d7234c5..41965c9d9 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -2,10 +2,7 @@ class VerifyEmailsJob < Que::Job def run(verification_id) email_address_verification = run_condition(EmailAddressVerification.find(verification_id)) - if email_address_verification.recently_verified? - destroy - return - end + return if email_address_verification.recently_verified? ActiveRecord::Base.transaction do email_address_verification.verify diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index de8041c32..f7c9e59fb 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -3,15 +3,15 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_or_create_by(email: email, - domain: domain(email)) + EmailAddressVerification.find_or_create_by(email: email.downcase, + domain: domain(email.downcase)) end def billing_email_verification return unless attribute_names.include?('billing_email') - EmailAddressVerification.find_or_create_by(email: billing_email, - domain: domain(email)) + EmailAddressVerification.find_or_create_by(email: billing_email.downcase, + domain: domain(email.downcase)) end def domain(email) diff --git a/config/schedule.rb b/config/schedule.rb index 089ce93f9..7ebf97d12 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -62,6 +62,10 @@ if @cron_group == 'registry' rake 'domain:discard' end + every 10.minutes do + rake 'verify_email:all_domains' + end + # Should be at least once every 4 days, since according to LHV specs: # "Unread messages older than 5 days are automatically scheduled for deletion" # https://partners.lhv.ee/en/connect/#messaging diff --git a/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb b/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb new file mode 100644 index 000000000..a7e2f8ee8 --- /dev/null +++ b/db/migrate/20200610090110_change_email_verification_fields_to_citext.rb @@ -0,0 +1,13 @@ +class ChangeEmailVerificationFieldsToCitext < ActiveRecord::Migration[6.0] + def up + enable_extension 'citext' + change_column :email_address_verifications, :email, :citext + change_column :email_address_verifications, :domain, :citext + end + + def down + change_column :email_address_verifications, :email, :string + change_column :email_address_verifications, :domain, :string + disable_extension 'citext' + end +end diff --git a/db/structure.sql b/db/structure.sql index c5ad902b7..9db7d07d1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -33,10 +33,17 @@ CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -- --- Name: EXTENSION btree_gist; Type: COMMENT; Schema: -; Owner: - +-- Name: citext; Type: EXTENSION; Schema: -; Owner: - -- -COMMENT ON EXTENSION btree_gist IS 'support for indexing common datatypes in GiST'; +CREATE EXTENSION IF NOT EXISTS citext WITH SCHEMA public; + + +-- +-- Name: EXTENSION citext; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION citext IS 'data type for case-insensitive character strings'; -- @@ -822,10 +829,10 @@ ALTER SEQUENCE public.domains_id_seq OWNED BY public.domains.id; CREATE TABLE public.email_address_verifications ( id bigint NOT NULL, - email character varying NOT NULL, + email public.citext NOT NULL, verified_at timestamp without time zone, success boolean DEFAULT false NOT NULL, - domain character varying NOT NULL + domain public.citext NOT NULL ); @@ -4700,5 +4707,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20200518104105'), ('20200529115011'), ('20200605100827'), +('20200610090110'), ('20200630081231'); From 56ac816dd1aa36ad246d682663764f5ac331749d Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Fri, 12 Jun 2020 15:37:53 +0500 Subject: [PATCH 20/26] Add thirs status - verification failed --- app/controllers/admin/contacts_controller.rb | 2 +- app/helpers/application_helper.rb | 10 ++++ app/models/contact.rb | 7 ++- app/models/email_address_verification.rb | 22 +++++++-- app/models/registrar.rb | 10 ---- app/views/admin/contacts/index.haml | 7 ++- .../admin/contacts/partials/_general.haml | 3 +- app/views/admin/registrars/index.html.erb | 8 +-- .../admin/registrars/show/_billing.html.erb | 4 +- .../admin/registrars/show/_contacts.html.erb | 4 +- ...20200608084321_fill_email_verifications.rb | 6 +-- lib/tasks/verify_email.rake | 4 +- test/tasks/emails/verify_email_task_test.rb | 49 +++++++++++++++++++ 13 files changed, 96 insertions(+), 40 deletions(-) create mode 100644 test/tasks/emails/verify_email_task_test.rb diff --git a/app/controllers/admin/contacts_controller.rb b/app/controllers/admin/contacts_controller.rb index 685a08c86..793fa1209 100644 --- a/app/controllers/admin/contacts_controller.rb +++ b/app/controllers/admin/contacts_controller.rb @@ -30,7 +30,7 @@ module Admin if params[:only_no_country_code].eql?('1') contacts = contacts.where("ident_country_code is null or ident_country_code=''") end - contacts = contacts.email_not_verified if params[:email_not_verified].eql?('1') + contacts = contacts.email_verification_failed if params[:email_verification_failed].eql?('1') contacts end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 8de3fdc70..5c742afce 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -108,4 +108,14 @@ module ApplicationHelper def body_css_class [controller_path.split('/').map!(&:dasherize), action_name.dasherize, 'page'].join('-') end + + def verified_email_span(verification) + content_tag(:span, verification.email, class: verified_email_class(verification)) + end + + def verified_email_class(verification) + return 'text-danger' if verification.failed? + return 'text-primary' if verification.not_verified? + return 'text-success' if verification.verified? + end end diff --git a/app/models/contact.rb b/app/models/contact.rb index a24a53a72..ac64b059f 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -23,10 +23,9 @@ class Contact < ApplicationRecord accepts_nested_attributes_for :legal_documents - scope :email_not_verified, lambda { - joins('LEFT JOIN :email_address_verifications emv ON contacts.email = emv.email') - .where('verified_at IS NULL OR verified_at <= ?', - EmailAddressVerification.verification_period) + scope :email_verification_failed, lambda { + joins('LEFT JOIN email_address_verifications emv ON contacts.email = emv.email') + .where('success = false and verified_at IS NOT NULL') } validates :name, :email, presence: true diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 412ce7f83..b478ab0f8 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -6,7 +6,11 @@ class EmailAddressVerification < ApplicationRecord } scope :verified_recently, lambda { - where('verified_at IS NOT NULL and verified_at >= ?', verification_period) + where('verified_at IS NOT NULL and verified_at >= ?', verification_period).where(success: true) + } + + scope :verification_failed, lambda { + where.not(verified_at: nil).where(success: false) } def recently_verified? @@ -22,16 +26,28 @@ class EmailAddressVerification < ApplicationRecord Time.zone.now - RECENTLY_VERIFIED_PERIOD end + def not_verified? + verified_at.blank? && !success + end + + def failed? + verified_at.present? && !success + end + + def verified? + success + end + def verify # media = success ? :mx : :smtp - media = :mx + media = :regex validation_request = Truemail.validate(email, with: media) if validation_request.result.success update(verified_at: Time.zone.now, success: true) else - update(verified_at: nil, + update(verified_at: Time.zone.now, success: false) end diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 9abdfdb68..470d768b7 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -13,16 +13,6 @@ class Registrar < ApplicationRecord has_many :nameservers, through: :domains has_many :whois_records has_many :white_ips, dependent: :destroy - # belongs_to :email_address_verification, class_name: 'EmailAddressVerification', - # primary_key: 'email', - # foreign_key: 'email', - # optional: true, - # inverse_of: :registrar - # belongs_to :billing_email_address_verification, class_name: 'EmailAddressVerification', - # primary_key: 'email', - # foreign_key: 'billing_email', - # optional: true, - # inverse_of: :billing_registrar delegate :balance, to: :cash_account, allow_nil: true diff --git a/app/views/admin/contacts/index.haml b/app/views/admin/contacts/index.haml index bf3665664..cbd11d3fc 100644 --- a/app/views/admin/contacts/index.haml +++ b/app/views/admin/contacts/index.haml @@ -65,8 +65,8 @@ = check_box_tag :only_no_country_code, '1',params[:only_no_country_code].eql?('1'), style: 'width:auto;height:auto;float:right' .col-md-3 .form-group - = label_tag :email_not_verified, "Email not verified" - = check_box_tag :email_not_verified, '1',params[:email_not_verified].eql?('1'), style: 'width:auto;height:auto;float:right' + = label_tag :email_verification_failed, "Email verification failed" + = check_box_tag :email_verification_failed, '1',params[:email_verification_failed].eql?('1'), style: 'width:auto;height:auto;float:right' .row .col-md-3{style: 'padding-top: 25px;float:right;'} @@ -100,8 +100,7 @@ %td= link_to(contact, admin_contact_path(contact)) %td= contact.code %td= ident_for(contact) - %td{class: ('text-danger' unless contact.email_verification.success)} - = contact.email + %td= verified_email_span(contact.email_verification) %td= l(contact.created_at, format: :short) %td - if contact.registrar diff --git a/app/views/admin/contacts/partials/_general.haml b/app/views/admin/contacts/partials/_general.haml index c4a151294..6568cd3d0 100644 --- a/app/views/admin/contacts/partials/_general.haml +++ b/app/views/admin/contacts/partials/_general.haml @@ -17,8 +17,7 @@ %dd= ident_for(@contact) %dt= t(:email) - %dd{class: ('text-danger' unless @contact.email_verification&.success)} - = @contact.email + %dd= verified_email_span(@contact.email_verification) %dt= t(:phone) %dd= @contact.phone diff --git a/app/views/admin/registrars/index.html.erb b/app/views/admin/registrars/index.html.erb index 6ed864972..e641f5294 100644 --- a/app/views/admin/registrars/index.html.erb +++ b/app/views/admin/registrars/index.html.erb @@ -49,13 +49,9 @@ <%= "#{x.test_registrar}" %> - > - <%= "#{x.email}" %> - + <%= verified_email_span(x.email_verification) %> <% if x[:billing_email].present? %> - > - <%= "#{x[:billing_email]}" %> - + <%= verified_email_span(x.billing_email_verification) %> <% end %> diff --git a/app/views/admin/registrars/show/_billing.html.erb b/app/views/admin/registrars/show/_billing.html.erb index ab731ce1d..07bccc7f4 100644 --- a/app/views/admin/registrars/show/_billing.html.erb +++ b/app/views/admin/registrars/show/_billing.html.erb @@ -15,8 +15,8 @@
<%= registrar.accounting_customer_code %>
<%= Registrar.human_attribute_name :billing_email %>
-
> - <%= registrar.billing_email %> +
+ <%= verified_email_span(registrar.billing_email_verification) %>
<%= Registrar.human_attribute_name :reference_no %>
diff --git a/app/views/admin/registrars/show/_contacts.html.erb b/app/views/admin/registrars/show/_contacts.html.erb index b5a222dd5..0ca1158d3 100644 --- a/app/views/admin/registrars/show/_contacts.html.erb +++ b/app/views/admin/registrars/show/_contacts.html.erb @@ -15,8 +15,8 @@
<%= @registrar.phone %>
<%= Registrar.human_attribute_name :email %>
-
> - <%= @registrar.email %> +
+ <%= verified_email_span(@registrar.email_verification) %>
diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb index 68f02b133..073bf2b24 100644 --- a/db/data/20200608084321_fill_email_verifications.rb +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -1,8 +1,8 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] def up - registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?) - registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?) - contact_emails = Contact.pluck(:email).uniq.reject(&:blank?) + registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?).map(&:downcase) + registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?).map(&:downcase) + contact_emails = Contact.pluck(:email).uniq.reject(&:blank?).map(&:downcase) emails = (contact_emails + registrar_emails + registrar_billing_emails).uniq diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index d069100f2..68dda6e60 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -2,9 +2,7 @@ namespace :verify_email do desc 'Stars verifying email jobs' task all_domains: :environment do verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) - verifications_by_domain.each do |domain, verifications| - next if domain == 'not_found' - + verifications_by_domain.each do |_domain, verifications| ver = verifications.sample # Verify random email to not to clog the SMTP servers VerifyEmailsJob.enqueue(ver.id) next diff --git a/test/tasks/emails/verify_email_task_test.rb b/test/tasks/emails/verify_email_task_test.rb new file mode 100644 index 000000000..ff70730fd --- /dev/null +++ b/test/tasks/emails/verify_email_task_test.rb @@ -0,0 +1,49 @@ +require 'test_helper' + +class VerifyEmailTaskTest < ActiveSupport::TestCase + + def setup + @contact = contacts(:john) + @invalid_contact = contacts(:invalid_email) + @contact_verification = @contact.email_verification + @invalid_contact_verification = @invalid_contact.email_verification + + @default_whitelist = Truemail.configure.whitelisted_domains + @default_blacklist = Truemail.configure.blacklisted_domains + Truemail.configure.whitelisted_domains = whitelisted_domains + Truemail.configure.blacklisted_domains = blacklisted_domains + end + + def teardown + Truemail.configure.whitelisted_domains = @default_whitelist + Truemail.configure.blacklisted_domains = @default_blacklist + end + + def domain(email) + Mail::Address.new(email).domain + rescue Mail::Field::IncompleteParseError + nil + end + + def whitelisted_domains + [domain(@contact.email)].reject(&:blank?) + end + + def blacklisted_domains + [domain(@invalid_contact.email)].reject(&:blank?) + end + + def test_tasks_verifies_emails + capture_io { run_task } + + @contact_verification.reload + @invalid_contact_verification.reload + + assert @contact_verification.verified? + assert @invalid_contact_verification.failed? + end + + def run_task + Rake::Task['verify_email:all_domains'].execute + end +end From b4369bdcd0996a37ee959e30161b0aa345b6eec1 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Mon, 15 Jun 2020 15:57:44 +0500 Subject: [PATCH 21/26] Add support of punycode domains in emails --- app/models/concerns/email_verifable.rb | 47 ++++++++++++++++--- app/models/email_address_verification.rb | 3 +- config/initializers/truemail.rb | 4 +- ...20200608084321_fill_email_verifications.rb | 19 ++++---- test/models/contact_test.rb | 10 ++++ test/models/registrar_test.rb | 15 ++++++ 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index f7c9e59fb..ea55ad3b9 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -3,21 +3,54 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_or_create_by(email: email.downcase, - domain: domain(email.downcase)) + EmailAddressVerification.find_or_create_by(email: self.class.punycode_to_unicode(email), + domain: domain(email)) end def billing_email_verification return unless attribute_names.include?('billing_email') - EmailAddressVerification.find_or_create_by(email: billing_email.downcase, - domain: domain(email.downcase)) + EmailAddressVerification.find_or_create_by(email: self.class + .punycode_to_unicode(billing_email), + domain: domain(billing_email)) + end + + class_methods do + def domain(email) + Mail::Address.new(email).domain&.downcase || 'not_found' + rescue Mail::Field::IncompleteParseError + 'not_found' + end + + def local(email) + Mail::Address.new(email).local&.downcase || email + rescue Mail::Field::IncompleteParseError + email + end + + def punycode_to_unicode(email) + return email if domain(email) == 'not_found' + + local = local(email) + domain = SimpleIDN.to_unicode(domain(email)) + "#{local}@#{domain}"&.downcase + end + + def unicode_to_punycode(email) + return email if domain(email) == 'not_found' + + local = local(email) + domain = SimpleIDN.to_ascii(domain(email)) + "#{local}@#{domain}"&.downcase + end end def domain(email) - Mail::Address.new(email).domain || 'not_found' - rescue Mail::Field::IncompleteParseError - 'not_found' + SimpleIDN.to_unicode(self.class.domain(email)) + end + + def punycode_to_unicode(email) + self.class.punycode_to_unicode(email) end def verify_email_mx_smtp(field:, email:) diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index b478ab0f8..5cc982da6 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -39,8 +39,7 @@ class EmailAddressVerification < ApplicationRecord end def verify - # media = success ? :mx : :smtp - media = :regex + media = :mx validation_request = Truemail.validate(email, with: media) if validation_request.result.success diff --git a/config/initializers/truemail.rb b/config/initializers/truemail.rb index 337ac8605..cfc8e14b1 100644 --- a/config/initializers/truemail.rb +++ b/config/initializers/truemail.rb @@ -30,8 +30,10 @@ Truemail.configure do |config| # Available validation types: :regex, :mx, :smtp if Rails.env.production? config.default_validation_type = :smtp - else + elsif Rails.env.test? config.default_validation_type = :regex + else + config.default_validation_type = :mx end # Optional parameter. You can predefine which type of validation will be used for domains. diff --git a/db/data/20200608084321_fill_email_verifications.rb b/db/data/20200608084321_fill_email_verifications.rb index 073bf2b24..37a7f275c 100644 --- a/db/data/20200608084321_fill_email_verifications.rb +++ b/db/data/20200608084321_fill_email_verifications.rb @@ -1,10 +1,13 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] - def up - registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?).map(&:downcase) - registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?).map(&:downcase) - contact_emails = Contact.pluck(:email).uniq.reject(&:blank?).map(&:downcase) + include Concerns::EmailVerifable - emails = (contact_emails + registrar_emails + registrar_billing_emails).uniq + def up + registrar_billing_emails = Registrar.pluck(:billing_email).uniq.reject(&:blank?) + registrar_emails = Registrar.pluck(:email).uniq.reject(&:blank?) + contact_emails = Contact.pluck(:email).uniq.reject(&:blank?) + + emails = (contact_emails + registrar_emails + registrar_billing_emails) + emails = emails.map{ |email| punycode_to_unicode(email) }.uniq result = emails.map do |email| { email: email, domain: domain(email) } @@ -15,10 +18,4 @@ class FillEmailVerifications < ActiveRecord::Migration[6.0] def down EmailAddressVerification.delete_all end - - def domain(email) - Mail::Address.new(email).domain || 'not_found' - rescue Mail::Field::IncompleteParseError - 'not_found' - end end diff --git a/test/models/contact_test.rb b/test/models/contact_test.rb index 77e959a95..a412df203 100644 --- a/test/models/contact_test.rb +++ b/test/models/contact_test.rb @@ -270,6 +270,16 @@ class ContactTest < ActiveSupport::TestCase assert_equal domain.whois_record.try(:json).try(:[], 'registrant'), @contact.name end + def test_creates_email_verification_in_unicode + unicode_email = 'suur@äri.ee' + punycode_email = Contact.unicode_to_punycode(unicode_email) + + @contact.email = punycode_email + @contact.save + + assert_equal @contact.email_verification.email, unicode_email + end + private def make_contact_free_of_domains_where_it_acts_as_a_registrant(contact) diff --git a/test/models/registrar_test.rb b/test/models/registrar_test.rb index 6db466e1c..5a497fd76 100644 --- a/test/models/registrar_test.rb +++ b/test/models/registrar_test.rb @@ -61,6 +61,21 @@ class RegistrarTest < ActiveSupport::TestCase end end + def test_creates_email_verification_in_unicode + unicode_email = 'suur@äri.ee' + punycode_email = Registrar.unicode_to_punycode(unicode_email) + unicode_billing_email = 'billing@äri.ee' + punycode_billing_email = Registrar.unicode_to_punycode(unicode_billing_email) + + registrar = valid_registrar + registrar.email = punycode_email + registrar.billing_email = punycode_billing_email + registrar.save + + assert_equal registrar.email_verification.email, unicode_email + assert_equal registrar.billing_email_verification.email, unicode_billing_email + end + def test_invalid_without_accounting_customer_code registrar = valid_registrar registrar.accounting_customer_code = '' From 57cd588eee5ab1de4646985b78f0d04e0e9bec0e Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 16 Jun 2020 13:41:35 +0500 Subject: [PATCH 22/26] Add email verification rake task for one domain only --- app/models/email_address_verification.rb | 2 ++ lib/tasks/verify_email.rake | 14 +++++++++++++- test/tasks/emails/verify_email_task_test.rb | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 5cc982da6..28bd6a2f9 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -13,6 +13,8 @@ class EmailAddressVerification < ApplicationRecord where.not(verified_at: nil).where(success: false) } + scope :by_domain, ->(domain_name) { where(domain: domain_name) } + def recently_verified? verified_at.present? && verified_at > verification_period diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index 68dda6e60..6b64e0765 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -1,5 +1,5 @@ namespace :verify_email do - desc 'Stars verifying email jobs' + desc 'Stars verifying email jobs for all the domain' task all_domains: :environment do verifications_by_domain = EmailAddressVerification.not_verified_recently.group_by(&:domain) verifications_by_domain.each do |_domain, verifications| @@ -8,4 +8,16 @@ namespace :verify_email do next end end + + # Need to be run like 'bundle exec rake verify_email:domain['gmail.com']' + # In zsh syntax will be 'bundle exec rake verify_email:domain\['gmail.com'\]' + # Default 'bundle exec rake verify_email:domain' wil use 'internet.ee' domain + desc 'Stars verifying email jobs for domain stated in argument' + task :domain, [:domain_name] => [:environment] do |_task, args| + args.with_defaults(domain_name: "internet.ee") + + verifications_by_domain = EmailAddressVerification.not_verified_recently + .by_domain(args[:domain_name]) + verifications_by_domain.map{ |ver| VerifyEmailsJob.enqueue(ver.id) } + end end diff --git a/test/tasks/emails/verify_email_task_test.rb b/test/tasks/emails/verify_email_task_test.rb index ff70730fd..7cca11845 100644 --- a/test/tasks/emails/verify_email_task_test.rb +++ b/test/tasks/emails/verify_email_task_test.rb @@ -43,7 +43,21 @@ class VerifyEmailTaskTest < ActiveSupport::TestCase assert @invalid_contact_verification.failed? end + def test_domain_task_verifies_for_one_domain + capture_io { run_single_domain_task(@contact_verification.domain) } + + @contact_verification.reload + @invalid_contact_verification.reload + + assert @contact_verification.verified? + assert @invalid_contact_verification.not_verified? + end + def run_task Rake::Task['verify_email:all_domains'].execute end + + def run_single_domain_task(domain) + Rake::Task["verify_email:domain"].invoke(domain) + end end From b0332a7abdc6e6608d2c5c74328267895c593309 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 16 Jun 2020 14:31:16 +0500 Subject: [PATCH 23/26] Add logging to email_verification.log --- app/jobs/verify_emails_job.rb | 2 +- lib/tasks/verify_email.rake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/jobs/verify_emails_job.rb b/app/jobs/verify_emails_job.rb index 41965c9d9..75f4b7d91 100644 --- a/app/jobs/verify_emails_job.rb +++ b/app/jobs/verify_emails_job.rb @@ -24,7 +24,7 @@ class VerifyEmailsJob < Que::Job end def logger - Rails.logger + @logger ||= Logger.new(Rails.root.join('log', 'email_verification.log')) end def log_success(verification) diff --git a/lib/tasks/verify_email.rake b/lib/tasks/verify_email.rake index 6b64e0765..d49bb38b9 100644 --- a/lib/tasks/verify_email.rake +++ b/lib/tasks/verify_email.rake @@ -14,10 +14,10 @@ namespace :verify_email do # Default 'bundle exec rake verify_email:domain' wil use 'internet.ee' domain desc 'Stars verifying email jobs for domain stated in argument' task :domain, [:domain_name] => [:environment] do |_task, args| - args.with_defaults(domain_name: "internet.ee") + args.with_defaults(domain_name: 'internet.ee') verifications_by_domain = EmailAddressVerification.not_verified_recently .by_domain(args[:domain_name]) - verifications_by_domain.map{ |ver| VerifyEmailsJob.enqueue(ver.id) } + verifications_by_domain.map { |ver| VerifyEmailsJob.enqueue(ver.id) } end end From b446cff3fd2e78ee2e0663d281fa84741bd9b609 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Tue, 16 Jun 2020 16:50:19 +0500 Subject: [PATCH 24/26] Add verification creation on email validation, add tests --- app/models/concerns/email_verifable.rb | 43 ++++++--- app/models/email_address_verification.rb | 5 +- config/locales/admin/email_verifable.en.yml | 5 + config/locales/admin/email_verifable.et.yml | 5 + test/models/contact_test.rb | 35 +++++-- test/models/registrar_test.rb | 102 +++++++++++++------- 6 files changed, 137 insertions(+), 58 deletions(-) create mode 100644 config/locales/admin/email_verifable.en.yml create mode 100644 config/locales/admin/email_verifable.et.yml diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index ea55ad3b9..22c030753 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -3,16 +3,16 @@ module Concerns extend ActiveSupport::Concern def email_verification - EmailAddressVerification.find_or_create_by(email: self.class.punycode_to_unicode(email), - domain: domain(email)) + @email_verification ||= EmailAddressVerification.find_or_create_by(email: unicode_email, + domain: domain(email)) end def billing_email_verification return unless attribute_names.include?('billing_email') - EmailAddressVerification.find_or_create_by(email: self.class - .punycode_to_unicode(billing_email), - domain: domain(billing_email)) + @billing_email_verification ||= EmailAddressVerification + .find_or_create_by(email: unicode_billing_email, + domain: domain(billing_email)) end class_methods do @@ -45,6 +45,14 @@ module Concerns end end + def unicode_billing_email + self.class.punycode_to_unicode(billing_email) + end + + def unicode_email + self.class.punycode_to_unicode(email) + end + def domain(email) SimpleIDN.to_unicode(self.class.domain(email)) end @@ -53,16 +61,29 @@ module Concerns self.class.punycode_to_unicode(email) end - def verify_email_mx_smtp(field:, email:) - errors.add(field, :invalid) unless email.blank? || Truemail.valid?(email) - end - def correct_email_format - verify_email_mx_smtp(field: :email, email: email) + return if email.blank? + + result = email_verification.verify + process_result(result: result, field: :email) end def correct_billing_email_format - verify_email_mx_smtp(field: :billing_email, email: billing_email) + return if email.blank? + + result = billing_email_verification.verify + process_result(result: result, field: :billing_email) + end + + def process_result(result:, field:) + case result[:errors].keys.first + when :smtp + errors.add(field, I18n.t('email.email_smtp_check_error')) + when :mx + errors.add(field, I18n.t('email.email_mx_check_error')) + when :regex + errors.add(field, I18n.t('email.email_regex_check_error')) + end end end end diff --git a/app/models/email_address_verification.rb b/app/models/email_address_verification.rb index 28bd6a2f9..2fe7c0dbe 100644 --- a/app/models/email_address_verification.rb +++ b/app/models/email_address_verification.rb @@ -41,8 +41,7 @@ class EmailAddressVerification < ApplicationRecord end def verify - media = :mx - validation_request = Truemail.validate(email, with: media) + validation_request = Truemail.validate(email) if validation_request.result.success update(verified_at: Time.zone.now, @@ -52,6 +51,6 @@ class EmailAddressVerification < ApplicationRecord success: false) end - validation_request.result.success + validation_request.result end end diff --git a/config/locales/admin/email_verifable.en.yml b/config/locales/admin/email_verifable.en.yml new file mode 100644 index 000000000..28a502ad9 --- /dev/null +++ b/config/locales/admin/email_verifable.en.yml @@ -0,0 +1,5 @@ +en: + email: + email_smtp_check_error: SMTP check error + email_mx_check_error: Mail domain not found + email_regex_check_error: Invalid format diff --git a/config/locales/admin/email_verifable.et.yml b/config/locales/admin/email_verifable.et.yml new file mode 100644 index 000000000..dfe82e15f --- /dev/null +++ b/config/locales/admin/email_verifable.et.yml @@ -0,0 +1,5 @@ +et: + email: + email_smtp_check_error: Eposti aadressi ei leitud (SMTP viga) + email_mx_check_error: Eposti aadressi domeeni ei leitud + email_regex_check_error: Eposti aadress on vigane diff --git a/test/models/contact_test.rb b/test/models/contact_test.rb index a412df203..0cef5950e 100644 --- a/test/models/contact_test.rb +++ b/test/models/contact_test.rb @@ -3,6 +3,11 @@ require 'test_helper' class ContactTest < ActiveSupport::TestCase setup do @contact = contacts(:john) + @old_validation_type = Truemail.configure.default_validation_type + end + + teardown do + Truemail.configure.default_validation_type = @old_validation_type end def test_valid_contact_fixture_is_valid @@ -61,27 +66,37 @@ class ContactTest < ActiveSupport::TestCase assert contact.invalid? end - def tests_email_mx_and_smtp - Truemail.configure do |config| - config.default_validation_type = :smtp - end - + def test_email_verification_valid contact = valid_contact contact.email = 'info@internet.ee' assert contact.valid? + end + def test_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + contact = valid_contact contact.email = 'somecrude1337joke@internet.ee' assert contact.invalid? + assert_equal I18n.t('email.email_smtp_check_error'), contact.errors.messages[:email].first + end - contact.email = 'some@strangesentence@internet.ee' - assert contact.invalid? + def test_email_verification_mx_error + Truemail.configure.default_validation_type = :mx + contact = valid_contact contact.email = 'somecrude31337joke@somestrange31337domain.ee' assert contact.invalid? + assert_equal I18n.t('email.email_mx_check_error'), contact.errors.messages[:email].first + end - Truemail.configure do |config| - config.default_validation_type = :regex - end + def test_email_verification_regex_error + Truemail.configure.default_validation_type = :regex + + contact = valid_contact + contact.email = 'some@strangesentence@internet.ee' + assert contact.invalid? + assert_equal I18n.t('email.email_regex_check_error'), contact.errors.messages[:email].first end def test_invalid_without_phone diff --git a/test/models/registrar_test.rb b/test/models/registrar_test.rb index 5a497fd76..e59dc0695 100644 --- a/test/models/registrar_test.rb +++ b/test/models/registrar_test.rb @@ -5,11 +5,13 @@ class RegistrarTest < ActiveSupport::TestCase @registrar = registrars(:bestnames) @original_default_language = Setting.default_language @original_days_to_keep_invoices_active = Setting.days_to_keep_invoices_active + @old_validation_type = Truemail.configure.default_validation_type end teardown do Setting.default_language = @original_default_language Setting.days_to_keep_invoices_active = @original_days_to_keep_invoices_active + Truemail.configure.default_validation_type = @old_validation_type end def test_valid_registrar_is_valid @@ -38,27 +40,82 @@ class RegistrarTest < ActiveSupport::TestCase assert registrar.invalid? end - def tests_email_mx_and_smtp - Truemail.configure do |config| - config.default_validation_type = :smtp - end - + def test_email_verification_valid registrar = valid_registrar registrar.email = 'info@internet.ee' + registrar.billing_email = nil + assert registrar.valid? + end + def test_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + registrar = valid_registrar registrar.email = 'somecrude1337joke@internet.ee' - assert registrar.invalid? + registrar.billing_email = nil - registrar.email = 'some@strangesentence@internet.ee' assert registrar.invalid? + assert_equal I18n.t('email.email_smtp_check_error'), registrar.errors.messages[:email].first + end + def test_email_verification_mx_error + Truemail.configure.default_validation_type = :mx + + registrar = valid_registrar registrar.email = 'somecrude31337joke@somestrange31337domain.ee' - assert registrar.invalid? + registrar.billing_email = nil - Truemail.configure do |config| - config.default_validation_type = :regex - end + assert registrar.invalid? + assert_equal I18n.t('email.email_mx_check_error'), registrar.errors.messages[:email].first + end + + def test_email_verification_regex_error + Truemail.configure.default_validation_type = :regex + + registrar = valid_registrar + registrar.email = 'some@strangesentence@internet.ee' + registrar.billing_email = nil + + assert registrar.invalid? + assert_equal I18n.t('email.email_regex_check_error'), registrar.errors.messages[:email].first + end + + def test_billing_email_verification_valid + registrar = valid_registrar + registrar.billing_email = 'info@internet.ee' + + assert registrar.valid? + end + + def test_billing_email_verification_smtp_error + Truemail.configure.default_validation_type = :smtp + + registrar = valid_registrar + registrar.billing_email = 'somecrude1337joke@internet.ee' + + assert registrar.invalid? + assert_equal I18n.t('email.email_smtp_check_error'), registrar.errors.messages[:billing_email].first + end + + def test_billing_email_verification_mx_error + Truemail.configure.default_validation_type = :mx + + registrar = valid_registrar + registrar.billing_email = 'somecrude31337joke@somestrange31337domain.ee' + + assert registrar.invalid? + assert_equal I18n.t('email.email_mx_check_error'), registrar.errors.messages[:billing_email].first + end + + def test_billing_email_verification_regex_error + Truemail.configure.default_validation_type = :regex + + registrar = valid_registrar + registrar.billing_email = 'some@strangesentence@internet.ee' + + assert registrar.invalid? + assert_equal I18n.t('email.email_regex_check_error'), registrar.errors.messages[:billing_email].first end def test_creates_email_verification_in_unicode @@ -88,29 +145,6 @@ class RegistrarTest < ActiveSupport::TestCase assert registrar.valid? end - def tests_email_mx_and_smtp - Truemail.configure do |config| - config.default_validation_type = :smtp - end - - registrar = valid_registrar - registrar.billing_email = 'info@internet.ee' - assert registrar.valid? - - registrar.billing_email = 'somecrude1337joke@internet.ee' - assert registrar.invalid? - - registrar.billing_email = 'непонятное@словосочетание@internet.ee' - assert registrar.invalid? - - registrar.billing_email = 'somecrude31337joke@somestrange31337domain.ee' - assert registrar.invalid? - - Truemail.configure do |config| - config.default_validation_type = :regex - end - end - def test_returns_billing_email_when_provided billing_email = 'billing@registrar.test' registrar = Registrar.new(billing_email: billing_email) From 68d32568fcfa01e46f57d5140129b7114514fb18 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 17 Jun 2020 11:06:43 +0500 Subject: [PATCH 25/26] Fix I18n --- .travis.yml | 3 ++- app/models/concerns/email_verifable.rb | 6 +++--- config/locales/admin/email_verifable.en.yml | 2 +- config/locales/admin/email_verifable.et.yml | 2 +- test/models/contact_test.rb | 6 +++--- test/models/registrar_test.rb | 12 ++++++------ 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01373f29d..6994c551e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: ruby cache: bundler env: - DB=postgresql -bundler_args: --without development staging production before_install: - "wget -N http://chromedriver.storage.googleapis.com/2.43/chromedriver_linux64.zip -P ~/" - "unzip ~/chromedriver_linux64.zip -d ~/" @@ -10,6 +9,7 @@ before_install: - "sudo mv -f ~/chromedriver /usr/local/share/" - "sudo chmod +x /usr/local/share/chromedriver" - "sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver" + before_script: - "cp config/application.yml.sample config/application.yml" - "echo \"openssl_config_path: 'test/fixtures/files/test_ca/openssl.cnf'\" >> config/application.yml" @@ -19,6 +19,7 @@ before_script: - "echo \"ca_key_path: 'test/fixtures/files/test_ca/private/ca.key.pem'\" >> config/application.yml" - "echo \"ca_key_password: 'password'\" >> config/application.yml" - "cp config/database_travis.yml config/database.yml" + - "bundle config set without 'development staging production'" - "bundle exec rake db:setup:all" - "bundle exec rake data:migrate" - "curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter" diff --git a/app/models/concerns/email_verifable.rb b/app/models/concerns/email_verifable.rb index 22c030753..b4fbf74e3 100644 --- a/app/models/concerns/email_verifable.rb +++ b/app/models/concerns/email_verifable.rb @@ -78,11 +78,11 @@ module Concerns def process_result(result:, field:) case result[:errors].keys.first when :smtp - errors.add(field, I18n.t('email.email_smtp_check_error')) + errors.add(field, I18n.t('email_verifable.email_smtp_check_error')) when :mx - errors.add(field, I18n.t('email.email_mx_check_error')) + errors.add(field, I18n.t('email_verifable.email_mx_check_error')) when :regex - errors.add(field, I18n.t('email.email_regex_check_error')) + errors.add(field, I18n.t('email_verifable.email_regex_check_error')) end end end diff --git a/config/locales/admin/email_verifable.en.yml b/config/locales/admin/email_verifable.en.yml index 28a502ad9..724fa4c32 100644 --- a/config/locales/admin/email_verifable.en.yml +++ b/config/locales/admin/email_verifable.en.yml @@ -1,5 +1,5 @@ en: - email: + email_verifable: email_smtp_check_error: SMTP check error email_mx_check_error: Mail domain not found email_regex_check_error: Invalid format diff --git a/config/locales/admin/email_verifable.et.yml b/config/locales/admin/email_verifable.et.yml index dfe82e15f..6c008ed11 100644 --- a/config/locales/admin/email_verifable.et.yml +++ b/config/locales/admin/email_verifable.et.yml @@ -1,5 +1,5 @@ et: - email: + email_verifable: email_smtp_check_error: Eposti aadressi ei leitud (SMTP viga) email_mx_check_error: Eposti aadressi domeeni ei leitud email_regex_check_error: Eposti aadress on vigane diff --git a/test/models/contact_test.rb b/test/models/contact_test.rb index 0cef5950e..856019bfb 100644 --- a/test/models/contact_test.rb +++ b/test/models/contact_test.rb @@ -78,7 +78,7 @@ class ContactTest < ActiveSupport::TestCase contact = valid_contact contact.email = 'somecrude1337joke@internet.ee' assert contact.invalid? - assert_equal I18n.t('email.email_smtp_check_error'), contact.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_smtp_check_error'), contact.errors.messages[:email].first end def test_email_verification_mx_error @@ -87,7 +87,7 @@ class ContactTest < ActiveSupport::TestCase contact = valid_contact contact.email = 'somecrude31337joke@somestrange31337domain.ee' assert contact.invalid? - assert_equal I18n.t('email.email_mx_check_error'), contact.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_mx_check_error'), contact.errors.messages[:email].first end def test_email_verification_regex_error @@ -96,7 +96,7 @@ class ContactTest < ActiveSupport::TestCase contact = valid_contact contact.email = 'some@strangesentence@internet.ee' assert contact.invalid? - assert_equal I18n.t('email.email_regex_check_error'), contact.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_regex_check_error'), contact.errors.messages[:email].first end def test_invalid_without_phone diff --git a/test/models/registrar_test.rb b/test/models/registrar_test.rb index e59dc0695..8edbb1e4b 100644 --- a/test/models/registrar_test.rb +++ b/test/models/registrar_test.rb @@ -56,7 +56,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = nil assert registrar.invalid? - assert_equal I18n.t('email.email_smtp_check_error'), registrar.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_smtp_check_error'), registrar.errors.messages[:email].first end def test_email_verification_mx_error @@ -67,7 +67,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = nil assert registrar.invalid? - assert_equal I18n.t('email.email_mx_check_error'), registrar.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_mx_check_error'), registrar.errors.messages[:email].first end def test_email_verification_regex_error @@ -78,7 +78,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = nil assert registrar.invalid? - assert_equal I18n.t('email.email_regex_check_error'), registrar.errors.messages[:email].first + assert_equal I18n.t('email_verifable.email_regex_check_error'), registrar.errors.messages[:email].first end def test_billing_email_verification_valid @@ -95,7 +95,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = 'somecrude1337joke@internet.ee' assert registrar.invalid? - assert_equal I18n.t('email.email_smtp_check_error'), registrar.errors.messages[:billing_email].first + assert_equal I18n.t('email_verifable.email_smtp_check_error'), registrar.errors.messages[:billing_email].first end def test_billing_email_verification_mx_error @@ -105,7 +105,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = 'somecrude31337joke@somestrange31337domain.ee' assert registrar.invalid? - assert_equal I18n.t('email.email_mx_check_error'), registrar.errors.messages[:billing_email].first + assert_equal I18n.t('email_verifable.email_mx_check_error'), registrar.errors.messages[:billing_email].first end def test_billing_email_verification_regex_error @@ -115,7 +115,7 @@ class RegistrarTest < ActiveSupport::TestCase registrar.billing_email = 'some@strangesentence@internet.ee' assert registrar.invalid? - assert_equal I18n.t('email.email_regex_check_error'), registrar.errors.messages[:billing_email].first + assert_equal I18n.t('email_verifable.email_regex_check_error'), registrar.errors.messages[:billing_email].first end def test_creates_email_verification_in_unicode From 840f21c48fafcc50550dca7fe27b1f5cf413b024 Mon Sep 17 00:00:00 2001 From: Alex Sherman Date: Wed, 8 Jul 2020 17:20:43 +0500 Subject: [PATCH 26/26] Update Travis bundler command --- .travis.yml | 4 +- Gemfile | 2 +- Gemfile.lock | 87 +++++++++++++++++---------------- config/initializers/airbrake.rb | 2 +- db/structure.sql | 14 ------ 5 files changed, 49 insertions(+), 60 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6994c551e..bb74deecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,8 @@ before_install: - "sudo mv -f ~/chromedriver /usr/local/share/" - "sudo chmod +x /usr/local/share/chromedriver" - "sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver" - + - "bundle config set without 'development staging production'" + - "bundle config set deployment '[secure]'" before_script: - "cp config/application.yml.sample config/application.yml" - "echo \"openssl_config_path: 'test/fixtures/files/test_ca/openssl.cnf'\" >> config/application.yml" @@ -19,7 +20,6 @@ before_script: - "echo \"ca_key_path: 'test/fixtures/files/test_ca/private/ca.key.pem'\" >> config/application.yml" - "echo \"ca_key_password: 'password'\" >> config/application.yml" - "cp config/database_travis.yml config/database.yml" - - "bundle config set without 'development staging production'" - "bundle exec rake db:setup:all" - "bundle exec rake data:migrate" - "curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter" diff --git a/Gemfile b/Gemfile index f7556e166..99091f556 100644 --- a/Gemfile +++ b/Gemfile @@ -69,7 +69,7 @@ gem 'e_invoice', github: 'internetee/e_invoice', branch: :master gem 'lhv', github: 'internetee/lhv', branch: :master gem 'domain_name' gem 'haml', '~> 5.0' -gem 'wkhtmltopdf-binary' +gem 'wkhtmltopdf-binary', '~> 0.12.5.1' gem 'directo', github: 'internetee/directo', branch: 'master' diff --git a/Gemfile.lock b/Gemfile.lock index 0f3b2d355..e4ad26396 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,7 +9,7 @@ GIT GIT remote: https://github.com/internetee/directo.git - revision: bdfab4be20803c666dcefc9a9c607f915a056ac5 + revision: 8ff8a382d004ffb85722a6a7a68a020bd4d7159b branch: master specs: directo (1.0.1) @@ -112,6 +112,8 @@ GEM activerecord (6.0.3.2) activemodel (= 6.0.3.2) activesupport (= 6.0.3.2) + activerecord-import (1.0.5) + activerecord (>= 3.2) activestorage (6.0.3.2) actionpack (= 6.0.3.2) activejob (= 6.0.3.2) @@ -125,18 +127,18 @@ GEM zeitwerk (~> 2.2, >= 2.2.2) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) - airbrake (10.0.1) + airbrake (10.0.5) airbrake-ruby (~> 4.13) - airbrake-ruby (4.13.0) + airbrake-ruby (4.15.0) rbtree3 (~> 0.5) akami (1.3.1) gyoku (>= 0.4.0) nokogiri - autodoc (0.7.3) + autodoc (0.7.4) actionpack activesupport (>= 3.0.0) rspec - autoprefixer-rails (9.7.4) + autoprefixer-rails (9.8.4) execjs bcrypt (3.1.13) bootsnap (1.4.6) @@ -145,8 +147,8 @@ GEM autoprefixer-rails (>= 5.2.1) sassc (>= 2.0.0) builder (3.2.4) - cancancan (3.0.2) - capybara (3.31.0) + cancancan (3.1.0) + capybara (3.33.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) @@ -165,8 +167,8 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.1.6) - countries (3.0.0) - i18n_data (~> 0.8.0) + countries (3.0.1) + i18n_data (~> 0.10.0) sixarm_ruby_unaccent (~> 1.1) unicode_utils (~> 1.4) crack (0.4.3) @@ -176,22 +178,23 @@ GEM daemons-rails (1.2.1) daemons multi_json (~> 1.0) - data_migrate (6.2.0) + data_migrate (6.3.0) rails (>= 5.0) - database_cleaner (1.8.2) - devise (4.7.1) + database_cleaner (1.8.5) + devise (4.7.2) bcrypt (~> 3.0) orm_adapter (~> 0.1) railties (>= 4.1.0) responders warden (~> 1.2.3) - diff-lcs (1.3) + diff-lcs (1.4.4) docile (1.3.2) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - dry-configurable (0.9.0) + dry-configurable (0.11.6) concurrent-ruby (~> 1.0) dry-core (~> 0.4, >= 0.4.7) + dry-equalizer (~> 0.2) dry-container (0.7.2) concurrent-ruby (~> 1.0) dry-configurable (~> 0.1, >= 0.1.3) @@ -199,11 +202,11 @@ GEM concurrent-ruby (~> 1.0) dry-equalizer (0.3.0) dry-inflector (0.2.0) - dry-logic (1.0.5) + dry-logic (1.0.6) concurrent-ruby (~> 1.0) dry-core (~> 0.2) dry-equalizer (~> 0.2) - dry-types (1.2.2) + dry-types (1.4.0) concurrent-ruby (~> 1.0) dry-container (~> 0.3) dry-core (~> 0.4, >= 0.4.4) @@ -213,12 +216,12 @@ GEM erubi (1.9.0) erubis (2.7.0) execjs (2.7.0) - ffi (1.12.2) + ffi (1.13.1) figaro (1.1.1) thor (~> 0.14) globalid (0.4.2) activesupport (>= 4.2.0) - grape (1.3.0) + grape (1.3.3) activesupport builder dry-types (>= 1.1) @@ -230,7 +233,7 @@ GEM haml (5.1.2) temple (>= 0.8.0) tilt - hashdiff (1.0.0) + hashdiff (1.0.1) hpricot (0.8.6) http-accept (1.7.0) http-cookie (1.0.3) @@ -241,10 +244,10 @@ GEM socksify i18n (1.8.3) concurrent-ruby (~> 1.0) - i18n_data (0.8.0) + i18n_data (0.10.0) isikukood (0.1.2) iso8601 (0.12.1) - jquery-rails (4.3.5) + jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) @@ -264,7 +267,7 @@ GEM kaminari-core (= 1.2.1) kaminari-core (1.2.1) keystores (0.4.0) - libxml-ruby (3.1.0) + libxml-ruby (3.2.0) listen (3.2.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -279,7 +282,7 @@ GEM method_source (0.8.2) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2019.1009) + mime-types-data (3.2020.0512) mimemagic (0.3.5) mina (0.3.1) open4 (~> 1.3.4) @@ -289,7 +292,7 @@ GEM minitest (5.14.1) monetize (1.9.4) money (~> 6.12) - money (6.13.7) + money (6.13.8) i18n (>= 0.6.4, <= 2) money-rails (1.13.3) activesupport (>= 3.0) @@ -304,7 +307,7 @@ GEM mustermann (>= 1.0.0) netrc (0.11.0) nio4r (2.5.2) - nokogiri (1.10.9) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) nori (2.6.0) open4 (1.3.4) @@ -312,7 +315,7 @@ GEM paper_trail (10.3.1) activerecord (>= 4.2) request_store (~> 1.1) - pdfkit (0.8.4.1) + pdfkit (0.8.4.3.1) pg (1.2.2) polyamorous (2.3.2) activerecord (>= 5.2.1) @@ -320,7 +323,7 @@ GEM coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - public_suffix (4.0.3) + public_suffix (4.0.5) puma (4.3.5) nio4r (~> 2.0) que (0.14.3) @@ -375,10 +378,10 @@ GEM ffi (~> 1.0) rbtree3 (0.6.0) rdoc (4.3.0) - regexp_parser (1.6.0) + regexp_parser (1.7.1) request_store (1.5.0) rack (>= 1.4) - responders (3.0.0) + responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) rest-client (2.1.0) @@ -390,21 +393,21 @@ GEM rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) rspec-mocks (~> 3.9.0) - rspec-core (3.9.1) - rspec-support (~> 3.9.1) - rspec-expectations (3.9.0) + rspec-core (3.9.2) + rspec-support (~> 3.9.3) + rspec-expectations (3.9.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) rspec-mocks (3.9.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) - rspec-support (3.9.2) + rspec-support (3.9.3) ruby2_keywords (0.0.2) - rubyzip (2.2.0) + rubyzip (2.3.0) safe_yaml (1.0.5) sass-rails (6.0.0) sassc-rails (~> 2.1, >= 2.1.1) - sassc (2.2.1) + sassc (2.4.0) ffi (~> 1.9) sassc-rails (2.1.2) railties (>= 4.0.0) @@ -412,7 +415,7 @@ GEM sprockets (> 3.0) sprockets-rails tilt - savon (2.12.0) + savon (2.12.1) akami (~> 1.2) builder (>= 2.1.2) gyoku (~> 1.2) @@ -455,7 +458,7 @@ GEM thor (0.20.3) thread_safe (0.3.6) tilt (2.0.10) - truemail (1.7.1) + truemail (1.8.0) simpleidn (~> 0.1.1) tzinfo (1.2.7) thread_safe (~> 0.1) @@ -463,7 +466,7 @@ GEM execjs (>= 0.3.0, < 3) unf (0.1.4) unf_ext - unf_ext (0.0.7.6) + unf_ext (0.0.7.7) unicode_utils (1.4.0) validates_email_format_of (1.6.3) i18n @@ -472,11 +475,11 @@ GEM wasabi (3.5.0) httpi (~> 2.0) nokogiri (>= 1.4.2) - webdrivers (4.2.0) + webdrivers (4.4.1) nokogiri (~> 1.6) rubyzip (>= 1.3.0) selenium-webdriver (>= 3.0, < 4.0) - webmock (3.8.0) + webmock (3.8.3) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) @@ -485,7 +488,7 @@ GEM websocket-extensions (0.1.5) whenever (0.9.4) chronic (>= 0.6.3) - wkhtmltopdf-binary (0.12.5.1) + wkhtmltopdf-binary (0.12.5.4) xpath (3.2.0) nokogiri (~> 1.8) zeitwerk (2.3.1) @@ -553,7 +556,7 @@ DEPENDENCIES webdrivers webmock whenever (= 0.9.4) - wkhtmltopdf-binary + wkhtmltopdf-binary (~> 0.12.5.1) BUNDLED WITH 2.1.4 diff --git a/config/initializers/airbrake.rb b/config/initializers/airbrake.rb index 5c1983369..abfe408c1 100644 --- a/config/initializers/airbrake.rb +++ b/config/initializers/airbrake.rb @@ -17,5 +17,5 @@ Airbrake.configure do |config| end config.environment = ENV['airbrake_env'] || Rails.env config.ignore_environments = %w[test] - config.blacklist_keys = Rails.application.config.filter_parameters + config.blocklist_keys = Rails.application.config.filter_parameters end diff --git a/db/structure.sql b/db/structure.sql index 9db7d07d1..587f68be8 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -3635,20 +3635,6 @@ CREATE INDEX index_domains_on_statuses ON public.domains USING gin (statuses); CREATE INDEX index_email_address_verifications_on_domain ON public.email_address_verifications USING btree (domain); --- --- Name: index_email_address_verifications_on_email; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX index_email_address_verifications_on_email ON public.email_address_verifications USING btree (email); - - --- --- Name: index_email_addresses_validations_on_email; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX index_email_addresses_validations_on_email ON public.email_addresses_validations USING btree (email); - - -- -- Name: index_epp_sessions_on_updated_at; Type: INDEX; Schema: public; Owner: - --