Merge remote-tracking branch 'origin/master' into 1556-get-rid-of-th6

CHANGELOG.md

@@ -1,3 +1,9 @@
+04.09.2020
+* Removed reduntant domains.registered_at db column [#1445](https://github.com/internetee/registry/pull/1445)
+* Certificate revocation lists are now hanlded outside of the application code [#1662](https://github.com/internetee/registry/pull/1662)
+* Monthly invoices are sent one by one to elliminate reply delay from accounting system [#1671](https://github.com/internetee/registry/pull/1671)
+* Fixed poll request ip whitelist issue [#1672](https://github.com/internetee/registry/pull/1672)
+
 03.09.2020
 * Refactored session timeout management [#711](https://github.com/internetee/registry/issues/711)
 * Improved error handling for epp requests without proper session [#1276](https://github.com/internetee/registry/pull/1276)

Gemfile.lock

@@ -9,7 +9,7 @@ GIT
 
 GIT
   remote: https://github.com/internetee/directo.git
-  revision: 8ff8a382d004ffb85722a6a7a68a020bd4d7159b
+  revision: e4ba54f601d1815fd8782a196788730d47861e86
   branch: master
   specs:
     directo (1.0.1)
@@ -165,7 +165,7 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.12.2)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     countries (3.0.1)
       i18n_data (~> 0.10.0)
       sixarm_ruby_unaccent (~> 1.1)
@@ -241,7 +241,7 @@ GEM
     httpi (2.4.4)
       rack
       socksify
-    i18n (1.8.3)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     i18n_data (0.10.0)
     isikukood (0.1.2)

app/jobs/directo_invoice_forward_job.rb

@@ -2,15 +2,16 @@ class DirectoInvoiceForwardJob < Que::Job
   def run(monthly: false, dry: false)
     @dry = dry
     (@month = Time.zone.now - 1.month) if monthly
-    api_url = ENV['directo_invoice_url']
-    sales_agent = Setting.directo_sales_agent
-    payment_term = Setting.directo_receipt_payment_term
-    @prepayment_product_id = Setting.directo_receipt_product_name
 
-    @client = DirectoApi::Client.new(api_url, sales_agent, payment_term)
+    @client = new_directo_client
     monthly ? send_monthly_invoices : send_receipts
   end
 
+  def new_directo_client
+    DirectoApi::Client.new(ENV['directo_invoice_url'], Setting.directo_sales_agent,
+                           Setting.directo_receipt_payment_term)
+  end
+
   def send_receipts
     unsent_invoices = Invoice.where(in_directo: false).non_cancelled
 
@@ -28,19 +29,18 @@ class DirectoInvoiceForwardJob < Que::Job
 
   def send_monthly_invoices
     Registrar.where.not(test_registrar: true).find_each do |registrar|
-      fetch_monthly_summary(registrar: registrar)
+      next unless registrar.cash_account
+
+      @client = new_directo_client
+      send_invoice_for_registrar(registrar)
     end
-
-    return unless @client.invoices.count.positive?
-
-    sync_with_directo
   end
 
-  def fetch_monthly_summary(registrar:)
+  def send_invoice_for_registrar(registrar)
-    return unless registrar.cash_account
-
     summary = registrar.monthly_summary(month: @month)
     @client.invoices.add_with_schema(invoice: summary, schema: 'summary') unless summary.nil?
+
+    sync_with_directo if @client.invoices.count.positive?
   end
 
   def assign_monthly_numbers

app/models/ability.rb

@@ -32,12 +32,14 @@ class Ability
 
   def epp # Registrar/api_user dynamic role
     if @user.registrar.api_ip_white?(@ip)
-      can :manage, :poll
       can :manage, Depp::Contact
       can :manage, :xml_console
       can :manage,   Depp::Domain
     end
 
+    # Poll
+    can :manage, :poll
+
     # REPP
     can(:manage, :repp)
 

app/models/certificate.rb

@@ -127,77 +127,20 @@ class Certificate < ApplicationRecord
       return false
     end
 
-    self.class.update_registry_crl
+    self.class.update_crl
-    self.class.reload_apache
     self
   end
 
   class << self
+    def tostdout(message)
+      time = Time.zone.now.utc
+      STDOUT << "#{time} - #{message}\n" unless Rails.env.test?
+    end
+
     def update_crl
-      update_id_crl
+      tostdout('Running crlupdater')
-      update_registry_crl
+      system('/bin/bash', ENV['crl_updater_path'].to_s)
-      reload_apache
+      tostdout('Finished running crlupdater')
-    end
-
-    def update_id_crl
-      STDOUT << "#{Time.zone.now.utc} - Updating ID CRL\n" unless Rails.env.test?
-
-      _out, _err, _st = Open3.capture3("
-        mkdir -p #{ENV['crl_dir']}/crl-id-temp
-        cd #{ENV['crl_dir']}/crl-id-temp
-
-        wget https://sk.ee/crls/esteid/esteid2007.crl
-        wget https://sk.ee/crls/juur/crl.crl
-        wget https://sk.ee/crls/eeccrca/eeccrca.crl
-        wget https://sk.ee/repository/crls/esteid2011.crl
-
-        openssl crl -in esteid2007.crl -out esteid2007.crl -inform DER
-        openssl crl -in crl.crl -out crl.crl -inform DER
-        openssl crl -in eeccrca.crl -out eeccrca.crl -inform DER
-        openssl crl -in esteid2011.crl -out esteid2011.crl -inform DER
-
-        ln -s crl.crl `openssl crl -hash -noout -in crl.crl`.r0
-        ln -s esteid2007.crl `openssl crl -hash -noout -in esteid2007.crl`.r0
-        ln -s eeccrca.crl `openssl crl -hash -noout -in eeccrca.crl`.r0
-        ln -s esteid2011.crl `openssl crl -hash -noout -in esteid2011.crl`.r0
-
-        rm -rf #{ENV['crl_dir']}/*.crl #{ENV['crl_dir']}/*.r0
-
-        mv #{ENV['crl_dir']}/crl-id-temp/* #{ENV['crl_dir']}
-
-        rm -rf #{ENV['crl_dir']}/crl-id-temp
-      ")
-
-      STDOUT << "#{Time.zone.now.utc} - ID CRL updated\n" unless Rails.env.test?
-    end
-
-    def update_registry_crl
-      STDOUT << "#{Time.zone.now.utc} - Updating registry CRL\n" unless Rails.env.test?
-
-      _out, _err, _st = Open3.capture3("
-        mkdir -p #{ENV['crl_dir']}/crl-temp
-        cd #{ENV['crl_dir']}/crl-temp
-
-        openssl ca -config #{ENV['openssl_config_path']} -keyfile #{ENV['ca_key_path']} -cert \
-        #{ENV['ca_cert_path']} -gencrl -out #{ENV['crl_dir']}/crl-temp/crl.pem -key \
-        '#{ENV['ca_key_password']}' -batch
-
-        ln -s crl.pem `openssl crl -hash -noout -in crl.pem`.r1
-
-        rm -rf #{ENV['crl_dir']}/*.pem #{ENV['crl_dir']}/*.r1
-
-        mv #{ENV['crl_dir']}/crl-temp/* #{ENV['crl_dir']}
-
-        rm -rf #{ENV['crl_dir']}/crl-temp
-      ")
-
-      STDOUT << "#{Time.zone.now.utc} - Registry CRL updated\n" unless Rails.env.test?
-    end
-
-    def reload_apache
-      STDOUT << "#{Time.zone.now.utc} - Reloading apache\n" unless Rails.env.test?
-      _out, _err, _st = Open3.capture3("sudo /etc/init.d/apache2 reload")
-      STDOUT << "#{Time.zone.now.utc} - Apache reloaded\n" unless Rails.env.test?
     end
 
     def parse_md_from_string(crt)

app/models/domain.rb

@@ -18,6 +18,7 @@ class Domain < ApplicationRecord
   alias_attribute :on_hold_time, :outzone_at
   alias_attribute :outzone_time, :outzone_at
   alias_attribute :auth_info, :transfer_code # Old attribute name; for PaperTrail
+  alias_attribute :registered_at, :created_at
 
   # TODO: whois requests ip whitelist for full info for own domains and partial info for other domains
   # TODO: most inputs should be trimmed before validatation, probably some global logic?
@@ -627,7 +628,7 @@ class Domain < ApplicationRecord
   def as_json(_options)
     hash = super
     hash['auth_info'] = hash.delete('transfer_code') # API v1 requirement
-    hash['valid_from'] = hash['registered_at'] # API v1 requirement
+    hash['valid_from'] = hash['created_at'] # API v1 requirement
     hash.delete('statuses_before_force_delete')
     hash
   end

app/models/epp/domain.rb

@@ -41,7 +41,6 @@ class Epp::Domain < Domain
       domain = Epp::Domain.new
       domain.attributes = domain.attrs_from(frame, current_user)
       domain.attach_default_contacts
-      domain.registered_at = Time.zone.now
 
       period = domain.period.to_i
       plural_period_unit_name = (domain.period_unit == 'm' ? 'months' : 'years').to_sym
@@ -150,7 +149,6 @@ class Epp::Domain < Domain
 
     at[:name] = frame.css('name').text if new_record?
     at[:registrar_id] = current_user.registrar.try(:id)
-    at[:registered_at] = Time.zone.now if new_record?
 
     period = frame.css('period').text
     at[:period] = (period.to_i == 0) ? 1 : period.to_i

app/models/whois_record.rb

@@ -36,7 +36,7 @@ class WhoisRecord < ApplicationRecord
     h[:disclaimer] = disclaimer_text if disclaimer_text.present?
     h[:name]       = domain.name
     h[:status]     = domain.statuses.map { |x| status_map[x] || x }
-    h[:registered] = domain.registered_at.try(:to_s, :iso8601)
+    h[:registered] = domain.registered_at.iso8601
     h[:changed]    = domain.updated_at.try(:to_s, :iso8601)
     h[:expire]     = domain.valid_to.to_date.to_s
     h[:outzone]    = domain.outzone_at.try(:to_date).try(:to_s)

app/views/admin/domains/partials/_general.html.erb

@@ -10,7 +10,7 @@
             <dt><%= t(:name) %></dt>
             <dd><%= @domain.name %></dd>
 
-            <dt><%= t(:registered_at) %></dt>
+            <dt><%= Domain.human_attribute_name :registered_at %></dt>
             <dd><%= l(@domain.registered_at) %></dd>
 
             <dt><%= t(:registrar_name) %></dt>

app/views/registrant/domains/partials/_general.html.erb

@@ -10,7 +10,7 @@
             <dt><%= t(:name) %></dt>
             <dd><%= @domain.name %></dd>
 
-            <dt><%= t(:registered_at) %></dt>
+            <dt><%= Domain.human_attribute_name :registered_at %></dt>
             <dd><%= l(@domain.registered_at) %></dd>
 
             <dt><%= Registrar.model_name.human %></dt>

config/application.yml.sample

@@ -34,6 +34,7 @@ time_zone: 'Tallinn' # more zones by rake time:zones:all
 openssl_config_path: '/etc/ssl/openssl.cnf'
 crl_dir:     '/home/registry/registry/shared/ca/crl'
 crl_path:     '/home/registry/registry/shared/ca/crl/crl.pem'
+crl_updater_path: '/home/registry/registry/shared/ca/crl/crlupdater.sh'
 ca_cert_path: '/home/registry/registry/shared/ca/certs/ca.crt.pem'
 ca_key_path:  '/home/registry/registry/shared/ca/private/ca.key.pem'
 ca_key_password: 'your-root-key-password'

config/locales/en.yml

@@ -195,7 +195,6 @@ en:
   registrar_name: 'Registrar'
   owner: 'Registrant'
   domain_details: 'Domain details'
-  registered_at: 'Registered at'
   password: 'Password'
   valid_from: 'Valid from'
   general: 'General'

db/migrate/20191217013225_remove_domains_registered_at.rb

@@ -0,0 +1,5 @@
+class RemoveDomainsRegisteredAt < ActiveRecord::Migration[5.0]
+  def change
+    remove_column :domains, :registered_at
+  end
+end

db/structure.sql

@@ -778,7 +778,6 @@ CREATE TABLE public.domains (
     id integer NOT NULL,
     name character varying NOT NULL,
     registrar_id integer NOT NULL,
-    registered_at timestamp without time zone,
     valid_to timestamp without time zone NOT NULL,
     registrant_id integer NOT NULL,
     transfer_code character varying NOT NULL,
@@ -4828,6 +4827,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20191203083643'),
 ('20191206183853'),
 ('20191212133136'),
+('20191217013225'),
 ('20191219112434'),
 ('20191219124429'),
 ('20191227110904'),

test/fixtures/domains.yml

@@ -5,7 +5,6 @@ shop:
   registrar: bestnames
   registrant: john
   transfer_code: 65078d5
-  registered_at: <%= Time.zone.parse('2010-07-04').to_s(:db) %>
   valid_to: <%= Time.zone.parse('2010-07-05').to_s(:db) %>
   outzone_at: <%= Time.zone.parse('2010-07-06').to_s(:db) %>
   delete_date: 2010-07-07

test/jobs/directo_invoice_forward_job_test.rb

@@ -151,4 +151,44 @@ class DirectoInvoiceForwardJobTest < ActiveSupport::TestCase
       DirectoInvoiceForwardJob.run(monthly: true, dry: false)
     end
   end
+
+  def test_sends_each_monthly_invoice_separately
+    WebMock.reset!
+
+    activity = account_activities(:one)
+    price = billing_prices(:create_one_year)
+    price.update(duration: '3 years')
+    activity.update(activity_type: 'create', price: price)
+
+    # Creating account activity for second action
+    another_activity = activity.dup
+    another_activity.account = accounts(:two)
+
+    AccountActivity.skip_callback(:create, :after, :update_balance)
+    another_activity.created_at = Time.zone.parse('2010-07-05 10:00')
+    another_activity.save
+    AccountActivity.set_callback(:create, :after, :update_balance)
+
+    response = <<-XML
+    <?xml version="1.0" encoding="UTF-8"?>
+    <results>
+      <Result Type="0" Desc="OK" docid="309902" doctype="ARVE" submit="Invoices"/>
+    </results>
+    XML
+
+    first_registrar_stub = stub_request(:post, ENV['directo_invoice_url']).with do |request|
+      body = CGI.unescape(request.body)
+      (body.include? 'StartDate') && (body.include? 'EndDate') && (body.include? 'bestnames')
+    end.to_return(status: 200, body: response)
+
+    second_registrar_stub = stub_request(:post, ENV['directo_invoice_url']).with do |request|
+      body = CGI.unescape(request.body)
+      (body.include? 'StartDate') && (body.include? 'EndDate') && (body.include? 'goodnames')
+    end.to_return(status: 200, body: response)
+
+    DirectoInvoiceForwardJob.run(monthly: true, dry: false)
+
+    assert_requested first_registrar_stub
+    assert_requested second_registrar_stub
+  end
 end

test/models/domain_test.rb

@@ -425,6 +425,12 @@ class DomainTest < ActiveSupport::TestCase
     assert_not(@domain.force_delete_scheduled?)
   end
 
+  def test_aliases_registered_at_to_created_at
+    created_at = Time.zone.parse('2010-07-05 10:00')
+    domain = Domain.new(created_at: created_at)
+    assert_equal created_at, domain.registered_at
+  end
+
   private
 
   def valid_domain

test/system/registrant_area/domains/details_test.rb

@@ -12,7 +12,7 @@ class RegistrantAreaDomainDetailsTest < ApplicationSystemTestCase
     visit registrant_domain_url(@domain)
 
     assert_text 'Name shop.test'
-    assert_text "Registered at #{l Time.zone.parse('2010-07-04')}"
+    assert_text "Registered at #{l @domain.registered_at}"
     assert_link 'Best Names', href: registrant_registrar_path(@domain.registrar)
 
     assert_text 'Transfer code'
@@ -73,4 +73,4 @@ class RegistrantAreaDomainDetailsTest < ApplicationSystemTestCase
 
     assert_field nil, with: registrant_domain_update_confirm_url(@domain, token: 'a01')
   end
-end
+end