Merge branch '2004-update-domain-eis-schema-with-server-release-prohibited-status' of https://github.com/internetee/registry into 2004-update-domain-eis-schema-with-server-release-prohibited-status

CHANGELOG.md

@@ -1,3 +1,29 @@
+31.05.2021
+* Bump nokogiri to 1.11.6 [#2012](https://github.com/internetee/registry/pull/2012)
+* Bump cancancan to 3.2.2 [#2013](https://github.com/internetee/registry/pull/2013)
+* Bump e_invoice to 7832ef6 [#2010](https://github.com/internetee/registry/pull/2014)
+* Bump active_interaction to 4.0.1 [#2015](https://github.com/internetee/registry/pull/2015)
+
+28.05.2021
+* fix for legacy contact object errors in admin [#2010](https://github.com/internetee/registry/pull/2010)
+
+27.05.2021
+* fixed error handling on invalid date format for epp contact create [#2006](https://github.com/internetee/registry/issues/2006)
+* csync input file with puycode values [#2003](https://github.com/internetee/registry/issues/2003)
+* autoloading newest db schema file versions [#1976](https://github.com/internetee/registry/issues/1976)
+* ForceDelete notes are updated with additionally found email addresses [#1913](https://github.com/internetee/registry/issues/1913)
+
+24.05.2021
+* Bump puma to 5.3.2 [#1999](https://github.com/internetee/registry/pull/1999)
+* Bump nokogiri to 1.11.5 [#2000](https://github.com/internetee/registry/pull/2000)
+* BUmp truemail to 2.4.2 [#2001](https://github.com/internetee/registry/pull/2001)
+
+21.05.2021
+* Tech contacts do not receive expiration emails any more [#1996](https://github.com/internetee/registry/issues/1996)
+
+20.05.2021
+* Moved data migrations from whois project to registry [#1928](https://github.com/internetee/registry/issues/1928)
+
 19.05.2021
 * Fix for contact update via registrant portal [#1968](https://github.com/internetee/registry/pull/1968)
 * REPP is returning domain count with domain list for improved pagination handling [#1969](https://github.com/internetee/registry/pull/1969)

Gemfile

@@ -24,7 +24,7 @@ gem 'validates_email_format_of', '1.6.3' # validates email against RFC 2822 and
 
 # 0.7.3 is the latest for Rails 4.2, however, it is absent on Rubygems server
 # https://github.com/huacnlee/rails-settings-cached/issues/165
-gem 'nokogiri', '~> 1.11.4'
+gem 'nokogiri', '~> 1.11.6'
 
 # style
 gem 'bootstrap-sass', '~> 3.4'

Gemfile.lock

@@ -18,7 +18,7 @@ GIT
 
 GIT
   remote: https://github.com/internetee/e_invoice.git
-  revision: 1e83e0b9d502d0a96d7bbbf9235a6621591d4e66
+  revision: 7832ef67ae9168bbb6aaba67dce6a69424234ff4
   branch: master
   specs:
     e_invoice (0.1.0)
@@ -114,7 +114,7 @@ GEM
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    active_interaction (4.0.0)
+    active_interaction (4.0.1)
       activemodel (>= 5, < 7)
     activejob (6.1.3.2)
       activesupport (= 6.1.3.2)
@@ -172,7 +172,7 @@ GEM
       autoprefixer-rails (>= 5.2.1)
       sassc (>= 2.0.0)
     builder (3.2.4)
-    cancancan (3.2.1)
+    cancancan (3.2.2)
     capybara (3.35.3)
       addressable
       mini_mime (>= 0.1.3)
@@ -292,7 +292,8 @@ GEM
       nokogiri (~> 1)
       rake
     mini_mime (1.0.3)
-    mini_portile2 (2.5.1)
+    mini_portile2 (2.5.2)
+      net-ftp (~> 0.1)
     minitest (5.14.4)
     monetize (1.9.4)
       money (~> 6.12)
@@ -307,12 +308,16 @@ GEM
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
+    net-ftp (0.1.2)
+      net-protocol
+      time
+    net-protocol (0.1.0)
     netrc (0.11.0)
     nio4r (2.5.7)
-    nokogiri (1.11.4)
+    nokogiri (1.11.6)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    nokogiri (1.11.4-x86_64-linux)
+    nokogiri (1.11.6-x86_64-linux)
       racc (~> 1.4)
     nori (2.6.0)
     omniauth (1.9.1)
@@ -341,7 +346,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.6)
-    puma (5.3.1)
+    puma (5.3.2)
       nio4r (~> 2.0)
     que (0.14.3)
     que-web (0.7.2)
@@ -462,7 +467,8 @@ GEM
     temple (0.8.2)
     thor (1.1.0)
     tilt (2.0.10)
-    truemail (2.4.1)
+    time (0.1.0)
+    truemail (2.4.2)
       simpleidn (~> 0.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -547,7 +553,7 @@ DEPENDENCIES
   mimemagic (= 0.4.3)
   minitest (~> 5.14)
   money-rails
-  nokogiri (~> 1.11.4)
+  nokogiri (~> 1.11.6)
   omniauth
   omniauth-rails_csrf_protection
   omniauth-tara!

app/interactions/domains/force_delete_email/base.rb

@@ -11,11 +11,21 @@ module Domains
         domains = domain_contacts.map(&:domain).flatten +
                   Domain.where(registrant_id: registrant_ids)
 
-        domains.each { |domain| process_force_delete(domain) unless domain.force_delete_scheduled? }
+        domains.each do |domain|
+          before_execute_force_delete(domain)
+        end
       end
 
       private
 
+      def before_execute_force_delete(domain)
+        if domain.force_delete_scheduled? && !domain.status_notes[DomainStatus::FORCE_DELETE].nil?
+          added_additional_email_into_notes(domain)
+        else
+          process_force_delete(domain)
+        end
+      end
+
       def process_force_delete(domain)
         domain.schedule_force_delete(type: :soft,
                                      notify_by_email: true,
@@ -24,6 +34,13 @@ module Domains
         save_status_note(domain)
       end
 
+      def added_additional_email_into_notes(domain)
+        return if domain.status_notes[DomainStatus::FORCE_DELETE].include? email
+
+        domain.status_notes[DomainStatus::FORCE_DELETE].concat(' ' + email)
+        domain.save(validate: false)
+      end
+
       def save_status_note(domain)
         domain.status_notes[DomainStatus::FORCE_DELETE] = email
         domain.save(validate: false)

app/jobs/csync_job.rb

@@ -83,9 +83,9 @@ class CsyncJob < ApplicationJob
   # From this point we're working on generating input for cdnskey-scanner
   def gather_pollable_domains
     @logger.info 'CsyncJob Generate: Gathering current domain(s) data'
-    Nameserver.select(:hostname, :domain_id).all.each do |ns|
+    Nameserver.select(:hostname_puny, :domain_id).all.each do |ns|
       %i[secure insecure].each do |i|
-        @input_store[i][ns.hostname] = [] unless @input_store[i].key? ns.hostname
+        @input_store[i][ns.hostname_puny] = [] unless @input_store[i].key? ns.hostname_puny
       end
 
       append_domains_to_list(ns)
@@ -94,14 +94,16 @@ class CsyncJob < ApplicationJob
 
   def append_domains_to_list(nameserver)
     Domain.where(id: nameserver.domain_id).all.each do |domain|
-      @input_store[domain.dnskeys.any? ? :secure : :insecure][nameserver.hostname].push domain.name
+      key = domain.dnskeys.any? ? :secure : :insecure
+      hostname = nameserver.hostname_puny || nameserver.hostname
+      @input_store[key][hostname].push domain.name_puny
     end
   end
 
   def generate_scanner_input
     @logger.info 'CsyncJob Generate: Gathering current domain(s) data'
     gather_pollable_domains
-
+    check_directory
     out_file = File.new(ENV['cdns_scanner_input_file'], 'w+')
 
     %i[secure insecure].each do |state|
@@ -113,6 +115,15 @@ class CsyncJob < ApplicationJob
     @logger.info 'CsyncJob Generate: Finished writing output to ' + ENV['cdns_scanner_input_file']
   end
 
+  def check_directory
+    dirname = File.dirname(ENV['cdns_scanner_input_file'])
+
+    FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+    return if File.exist?(ENV['cdns_scanner_input_file'])
+
+    FileUtils.touch(ENV['cdns_scanner_input_file'])
+  end
+
   def create_input_lines(out_file, state)
     @input_store[state].keys.each do |nameserver|
       domains = @input_store[state][nameserver].join(' ')

app/models/domain.rb

@@ -671,9 +671,8 @@ class Domain < ApplicationRecord
   end
 
   def expired_domain_contact_emails
-    force_delete_contact_emails.reject do |email|
+    (primary_contact_emails +
-      BouncedMailAddress.where(email: email).count.positive?
+    ["info@#{name}", "#{prepared_domain_name}@#{name}"]).uniq
-    end
   end
 
   def all_related_emails

app/views/admin/domain_versions/archive.haml

@@ -67,7 +67,9 @@
                   - else
                     - contact = Contact.all_versions_for([domain.registrant_id], version.created_at).first
                     - if contact.nil? && ver = Version::ContactVersion.where(item_id: domain.registrant_id).last
-                      - contact = Contact.new(ver.object.to_h.merge(ver.object_changes.to_h.each_with_object({}) {|(k,v), o| o[k] = v.last }))
+                      - merged_obj = ver.object_changes.to_h.each_with_object({}) {|(k,v), o| o[k] = v.last }
+                      - result = ver.object.to_h.merge(merged_obj)&.slice(*Contact&.column_names)
+                      - contact = Contact.new(result)
                     = contact.try(:name)
                     = " ".html_safe
                     = "(#{t(:deleted)})"

app/views/epp/error.xml.builder

@@ -2,7 +2,7 @@ xml.epp_head do
   xml.response do
     @errors.each do |error|
       x = error&.options
-      next if x.empty? || x == { value: nil }
+      next if x.empty? || x == { value: nil } || x[:code].blank?
 
       xml.result('code' => x[:code]) do
         xml.msg(x[:msg], 'lang' => 'en')

db/migrate/20180515105348_add_contact_requests_table.rb

@@ -0,0 +1,21 @@
+class AddContactRequestsTable < ActiveRecord::Migration[5.0]
+  def change
+    # create_table :contact_requests do |t|
+    #   t.integer    :whois_record_id, null: false
+    #   t.string     :secret,          null: false
+    #   t.string     :email,           null: false
+    #   t.string     :name,            null: false
+    #   t.datetime   :valid_to,        null: false
+    #   t.string     :status,          null: false, default: 'new'
+    #   t.inet       :ip_address
+
+    #   t.timestamps null: false
+    # end
+
+    # add_foreign_key :contact_requests, :whois_records
+    # add_index :contact_requests, :secret, unique: true
+    # add_index :contact_requests, :email
+    # add_index :contact_requests, :ip_address
+    # add_index :contact_requests, :whois_record_id
+  end
+end

db/migrate/20180627115124_remove_foreign_key_constraint_from_contact_request.rb

@@ -0,0 +1,5 @@
+class RemoveForeignKeyConstraintFromContactRequest < ActiveRecord::Migration[5.0]
+  def change
+    # remove_foreign_key :contact_requests, :whois_records
+  end
+end

db/migrate/20181102124618_remove_whois_records_body.rb

@@ -0,0 +1,9 @@
+class RemoveWhoisRecordsBody < ActiveRecord::Migration[5.2]
+  def up
+    # remove_column :whois_records, :body
+  end
+
+  def down
+    # add_column :whois_records, :body, :text
+  end
+end

db/migrate/20200914073130_add_message_id_to_contact_request.rb

@@ -0,0 +1,5 @@
+class AddMessageIdToContactRequest < ActiveRecord::Migration[6.0]
+  def change
+    # add_column :contact_requests, :message_id, :string
+  end
+end

db/structure.sql

@@ -541,17 +541,19 @@ ALTER SEQUENCE public.certificates_id_seq OWNED BY public.certificates.id;
 --
 
 CREATE TABLE public.contact_requests (
-                                         id integer NOT NULL,
+
-                                         whois_record_id integer NOT NULL,
+    id integer NOT NULL,
-                                         secret character varying NOT NULL,
+    whois_record_id integer NOT NULL,
-                                         email character varying NOT NULL,
+    secret character varying NOT NULL,
-                                         name character varying NOT NULL,
+    email character varying NOT NULL,
-                                         valid_to timestamp without time zone NOT NULL,
+    name character varying NOT NULL,
-                                         status character varying DEFAULT 'new'::character varying NOT NULL,
+    valid_to timestamp without time zone NOT NULL,
-                                         ip_address inet,
+    status character varying DEFAULT 'new'::character varying NOT NULL,
-                                         created_at timestamp without time zone NOT NULL,
+    ip_address inet,
-                                         updated_at timestamp without time zone NOT NULL,
+    created_at timestamp without time zone NOT NULL,
-                                         message_id character varying
+    updated_at timestamp without time zone NOT NULL,
+    message_id character varying
+
 );
 
 
@@ -2639,7 +2641,6 @@ CREATE TABLE public.whois_records (
     id integer NOT NULL,
     domain_id integer,
     name character varying,
-    body text,
     json json,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
@@ -4975,11 +4976,13 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20180327151906'),
 ('20180331200125'),
 ('20180422154642'),
+('20180515105348'),
 ('20180612042234'),
 ('20180612042625'),
 ('20180612042953'),
 ('20180613030330'),
 ('20180613045614'),
+('20180627115124'),
 ('20180713154915'),
 ('20180801114403'),
 ('20180808064402'),
@@ -5004,6 +5007,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20181017154143'),
 ('20181017205123'),
 ('20181022100114'),
+('20181102124618'),
 ('20181108154921'),
 ('20181129150515'),
 ('20181212105100'),
@@ -5096,6 +5100,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20200908131554'),
 ('20200910085157'),
 ('20200910102028'),
+('20200914073130'),
 ('20200916125326'),
 ('20200917104213'),
 ('20200921084356'),

test/fixtures/nameservers.yml

@@ -1,5 +1,6 @@
 shop_ns1:
   hostname: ns1.bestnames.test
+  hostname_puny: ns1.bestnames.test
   ipv4:
     - 192.0.2.1
   ipv6:
@@ -8,6 +9,7 @@ shop_ns1:
 
 shop_ns2:
   hostname: ns2.bestnames.test
+  hostname_puny: ns2.bestnames.test
   ipv4:
     - 192.0.2.2
   ipv6:
@@ -16,6 +18,7 @@ shop_ns2:
 
 airport_ns1:
   hostname: ns1.bestnames.test
+  hostname_puny: ns1.bestnames.test
   ipv4:
     - 192.0.2.2
   ipv6:
@@ -24,6 +27,7 @@ airport_ns1:
 
 airport_ns2:
   hostname: ns2.bestnames.test
+  hostname_puny: ns2.bestnames.test
   ipv4:
     - 192.0.2.0
     - 192.0.2.3
@@ -34,4 +38,5 @@ airport_ns2:
 
 metro_ns1:
   hostname: ns1.bestnames.test
+  hostname_puny: ns1.bestnames.test
   domain: metro

test/integration/epp/contact/create/base_test.rb

@@ -41,6 +41,42 @@ class EppContactCreateBaseTest < EppTestCase
     assert_not_empty contact.code
   end
 
+  def test_responses_with_error_on_birthday
+    name = 'new'
+    email = 'new@registrar.test'
+    phone = '+1.2'
+
+    request_xml = <<-XML
+      <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+      <epp xmlns="https://epp.tld.ee/schema/epp-ee-1.0.xsd">
+        <command>
+          <create>
+            <contact:create xmlns:contact="https://epp.tld.ee/schema/contact-ee-1.1.xsd">
+              <contact:postalInfo>
+                <contact:name>#{name}</contact:name>
+              </contact:postalInfo>
+              <contact:voice>#{phone}</contact:voice>
+              <contact:email>#{email}</contact:email>
+            </contact:create>
+          </create>
+          <extension>
+            <eis:extdata xmlns:eis="https://epp.tld.ee/schema/eis-1.0.xsd">
+              <eis:ident type="birthday" cc="LV">31-12-2000</eis:ident>
+            </eis:extdata>
+          </extension>
+        </command>
+      </epp>
+    XML
+
+
+    assert_no_difference 'Contact.count' do
+      post epp_create_path, params: { frame: request_xml },
+          headers: { 'HTTP_COOKIE' => 'session=api_bestnames' }
+    end
+
+    assert_epp_response :parameter_value_syntax_error
+  end
+
   def test_responces_error_with_email_error
     name = 'new'
     email = 'new@registrar@test'

test/interactions/expire_period/process_expired_test.rb

@@ -0,0 +1,27 @@
+require 'test_helper'
+
+class StartTest < ActiveSupport::TestCase
+  include ActionMailer::TestHelper
+
+  setup do
+    @domain = domains(:shop)
+    @domain.update(expire_time: Time.zone.now - 1.day)
+    ActionMailer::Base.deliveries.clear
+  end
+
+  def test_expired_domain_contact_emails_should_not_contain_tech_contacts
+    contact_list = []
+    @domain.tech_contacts.each do |contact|
+      contact_list << contact.email
+    end
+
+    email_address = @domain.expired_domain_contact_emails
+    email = DomainExpireMailer.expired_soft(domain: @domain,
+                                            registrar: @domain.registrar,
+                                            email: email_address).deliver_now
+
+    email.to.each do |received|
+      assert_not contact_list.include? received
+    end
+  end
+end

test/jobs/csync_job_test.rb

@@ -6,6 +6,10 @@ class CsyncJobTest < ActiveSupport::TestCase
   setup do
     @dnskey = dnskeys(:one)
     @domain = domains(:shop)
+    dirname = File.dirname(ENV['cdns_scanner_input_file'])
+
+    FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+    FileUtils.touch(ENV['cdns_scanner_input_file']) unless File.exists?(ENV['cdns_scanner_input_file'])
   end
 
   def test_generates_input_file_for_cdnskey_scanner
@@ -19,6 +23,21 @@ class CsyncJobTest < ActiveSupport::TestCase
     assert_equal expected_contents, IO.read(ENV['cdns_scanner_input_file'])
   end
 
+  def test_generates_input_file_from_name_puny
+    @domain.update(name: 'pööriöö.ee', name_puny: 'xn--pri-snaaca.ee')
+    @domain.save(validate: false)
+    @nameserver = @domain.nameservers.first
+    @nameserver.update(hostname: 'täpiline.ee', hostname_puny: 'xn--theke1-bua.ee')
+    @domain.reload
+    @dnskey.update(domain: @domain)
+
+    expected_contents = "[secure]\nns2.bestnames.test #{@domain.name_puny}\n#{@nameserver.hostname_puny} #{@domain.name_puny}\n" \
+    "[insecure]\nns2.bestnames.test airport.test\nns1.bestnames.test airport.test metro.test\n"
+
+    CsyncJob.perform_now(generate: true)
+    assert_equal expected_contents, IO.read(ENV['cdns_scanner_input_file'])
+  end
+
   def test_creates_csync_record_when_new_cdnskey_discovered
     assert_nil @domain.csync_record
     CsyncJob.perform_now

test/models/domain/force_delete_test.rb

@@ -385,6 +385,37 @@ class ForceDeleteTest < ActionMailer::TestCase
     assert notification.text.include? asserted_text
   end
 
+  def test_domain_should_have_several_bounced_emails
+    @domain.update(valid_to: Time.zone.parse('2012-08-05'))
+    assert_not @domain.force_delete_scheduled?
+    travel_to Time.zone.parse('2010-07-05')
+    email_one = 'one@strangesentence@internet.ee'
+    email_two = 'two@strangesentence@internet.ee'
+    asserted_text_one = "Invalid email: #{email_one}"
+    asserted_text_two = "Invalid email: #{email_two}"
+
+    contact_one = @domain.admin_contacts.first
+    contact_one.update_attribute(:email, email_one)
+    contact_one.email_verification.verify
+
+    assert contact_one.email_verification_failed?
+
+    contact_two = @domain.admin_contacts.first
+    contact_two.update_attribute(:email, email_two)
+    contact_two.email_verification.verify
+
+    assert contact_one.email_verification_failed?
+
+    @domain.reload
+
+    assert @domain.force_delete_scheduled?
+    assert_equal 'invalid_email', @domain.template_name
+    assert_equal Date.parse('2010-09-19'), @domain.force_delete_date.to_date
+    assert_equal Date.parse('2010-08-05'), @domain.force_delete_start.to_date
+    assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_one
+    assert @domain.status_notes[DomainStatus::FORCE_DELETE].include? email_two
+  end
+
   def test_lifts_force_delete_if_contact_fixed
     @domain.update(valid_to: Time.zone.parse('2012-08-05'))
     assert_not @domain.force_delete_scheduled?

test/system/admin_area/contact_versions_test.rb

@@ -24,7 +24,8 @@ class ContactVersionsTest < ApplicationSystemTestCase
       INSERT INTO log_contacts (item_type, item_id, event, whodunnit, object,
       object_changes, created_at, session, children, ident_updated_at, uuid)
       VALUES ('Contact', 75, 'update', '1-AdminUser',
-      '{"id": 75, "code": "test_code", "auth_info": "8b4d462aa04194ca78840a", "registrar_id": #{@registrar.id}, "old_field": "value"}',
+      '{"id": 75, "code": "test_code", "auth_info": "8b4d462aa04194ca78840a", "registrar_id": #{@registrar.id}, "old_field": "value",
+        "legal_id": "123"}',
       '{"other_made_up_field": "value"}', 
       '2018-04-23 15:50:48.113491', '2018-04-23 12:44:56',
       '{"legal_documents":[null]}', null, null