Merge branch 'improve-registrar' into registry-267

# Conflicts:
#	db/schema-read-only.rb
#	db/structure.sql

CHANGELOG.md

@@ -1,3 +1,6 @@
+22.12.2016
+* Return business registry code and country for 'org' type registrants in WHOIS and Rest-WHOIS
+
 16.12.2016
 * Allow contact address processing to be configurable via admin
 * EPP XML schema namespace "urn:ietf:params:xml:ns:epp-1.0" replaced with "https://epp.tld.ee/schema/epp-ee-1.0.xsd"

app/controllers/admin/registrars_controller.rb

@@ -17,10 +17,15 @@ class Admin::RegistrarsController < AdminController
   def create
     @registrar = Registrar.new(registrar_params)
 
-    if @registrar.save
+    begin
+      @registrar.transaction do
+        @registrar.save!
+        @registrar.accounts.create!(account_type: Account::CASH, currency: 'EUR')
+      end
+
       flash[:notice] = I18n.t('registrar_added')
       redirect_to [:admin, @registrar]
-    else
+    rescue ActiveRecord::RecordInvalid
       flash.now[:alert] = I18n.t('failed_to_add_registrar')
       render 'new'
     end

app/models/contact.rb

@@ -15,6 +15,7 @@ class Contact < ActiveRecord::Base
   has_paper_trail class_name: "ContactVersion", meta: { children: :children_log }
 
   attr_accessor :legal_document_id
+  alias_attribute :kind, :ident_type
 
   accepts_nested_attributes_for :legal_documents
 
@@ -583,4 +584,9 @@ class Contact < ActiveRecord::Base
       self[attr_name.to_sym] = nil
     end
   end
+
+  def reg_no
+    return if priv?
+    ident
+  end
 end

app/models/domain_cron.rb

@@ -142,4 +142,9 @@ class DomainCron
     )
   end
 
+  def self.delete_legal_doc_duplicates
+    Rake::Task['legal_doc:remove_duplicates'].reenable
+    Rake::Task['legal_doc:remove_duplicates'].invoke
+  end
+
 end

app/models/legal_document.rb

@@ -1,4 +1,5 @@
 class LegalDocument < ActiveRecord::Base
+  cattr_accessor :explicitly_write_file
   include EppErrors
   MIN_BODY_SIZE = (1.37 * 3.kilobytes).ceil
 
@@ -16,7 +17,7 @@ class LegalDocument < ActiveRecord::Base
   validate :val_body_length, if: ->(file){ file.path.blank? && !Rails.env.staging?}
 
   before_create :add_creator
-  before_save   :save_to_filesystem
+  before_save   :save_to_filesystem, if: :body
 
   def epp_code_map
     {
@@ -32,22 +33,82 @@ class LegalDocument < ActiveRecord::Base
 
 
   def save_to_filesystem
-    loop do
+    binary = Base64.decode64(body)
-      rand = SecureRandom.random_number.to_s.last(4)
+    digest = Digest::SHA1.new.update(binary).to_s
-      next if rand.to_i == 0 || rand.length < 4
 
-      dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}"
+    loop do
-      FileUtils.mkdir_p(dir, mode: 0775)
+        rand = SecureRandom.random_number.to_s.last(4)
-      self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}"
+        next if rand.to_i == 0 || rand.length < 4
-      break unless File.file?(path)
+        dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}"
+        FileUtils.mkdir_p(dir, mode: 0775)
+        self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}"
+        break unless File.file?(path)
     end
 
-    File.open(path, 'wb') { |f| f.write(Base64.decode64(body)) } unless Rails.env.test?
+    File.open(path, 'wb') { |f| f.write(binary) } if !Rails.env.test? || self.class.explicitly_write_file
     self.path = path
+    self.checksum = digest
+  end
+
+  def calc_checksum
+    digest = Digest::SHA1.new
+    digest.update File.binread(path)
+    digest.to_s
   end
 
   def add_creator
     self.creator_str = ::PaperTrail.whodunnit
     true
   end
+
+
+  def self.remove_duplicates
+    start = Time.zone.now.to_f
+    puts '-----> Removing legal documents duplicates'
+    count = 0
+    modified = Array.new
+
+    LegalDocument.where(documentable_type: "Domain").where.not(checksum: [nil, ""]).find_each do |orig_legal|
+      next if modified.include?(orig_legal.checksum)
+      next if !File.exist?(orig_legal.path)
+      modified.push(orig_legal.checksum)
+
+      LegalDocument.where(documentable_type: "Domain", documentable_id: orig_legal.documentable_id).
+          where(checksum: orig_legal.checksum).
+          where.not(id: orig_legal.id).where.not(path: orig_legal.path).each do |new_legal|
+        unless modified.include?(orig_legal.id)
+          File.delete(new_legal.path) if File.exist?(new_legal.path)
+          new_legal.update(path: orig_legal.path)
+          count += 1
+          puts "File #{new_legal.path} has been removed by Domain #{new_legal.documentable_id}. Document id: #{new_legal.id}"
+        end
+      end
+
+      contact_ids = DomainVersion.where(item_id: orig_legal.documentable_id).distinct.
+          pluck("object->>'registrant_id'", "object_changes->>'registrant_id'",
+                "children->>'tech_contacts'", "children->>'admin_contacts'",
+                "tech_contact_ids", "admin_contact_ids").flatten.uniq
+      contact_ids = contact_ids.map{|id|
+        case id
+          when Hash
+            id["id"]
+          when String
+            JSON.parse(id) rescue id.to_i
+          else
+            id
+        end
+      }.flatten.compact.uniq
+      LegalDocument.where(documentable_type: "Contact", documentable_id: contact_ids).
+          where(checksum: orig_legal.checksum).where.not(path: orig_legal.path).each do |new_legal|
+        unless modified.include?(orig_legal.id)
+          File.delete(new_legal.path) if File.exist?(new_legal.path)
+          new_legal.update(path: orig_legal.path)
+          count += 1
+          puts "File #{new_legal.path} has been removed by Contact #{new_legal.documentable_id}. Document id: #{new_legal.id}"
+        end
+      end
+    end
+    puts "-----> Duplicates fixed for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds"
+
+  end
 end

app/models/registrar.rb

@@ -57,11 +57,6 @@ class Registrar < ActiveRecord::Base
     RegenerateRegistrarWhoisesJob.enqueue id
   end
 
-  after_create :create_cash_account
-  def create_cash_account
-    accounts.create(account_type: Account::CASH, currency: 'EUR')
-  end
-
   class << self
     def search_by_query(query)
       res = search(name_or_reg_no_cont: query).result

app/models/whois_record.rb

@@ -41,6 +41,8 @@ class WhoisRecord < ActiveRecord::Base
         'ok' => 'ok (paid and in zone)'
     }
 
+    registrant = domain.registrant
+
     @disclosed = []
     h[:name]       = domain.name
     h[:status]     = domain.statuses.map { |x| status_map[x] || x }
@@ -50,11 +52,17 @@ class WhoisRecord < ActiveRecord::Base
     h[:outzone]    = domain.outzone_at.try(:to_date).try(:to_s)
     h[:delete]     = [domain.delete_at, domain.force_delete_at].compact.min.try(:to_date).try(:to_s)
 
+    h[:registrant] = registrant.name
+    h[:registrant_kind] = registrant.kind
 
-    h[:registrant]       = domain.registrant.name
+    if registrant.org?
-    h[:email] = domain.registrant.email
+      h[:registrant_reg_no] = registrant.reg_no
-    @disclosed << [:email, domain.registrant.email]
+      h[:registrant_ident_country_code] = registrant.ident_country_code
-    h[:registrant_changed]          = domain.registrant.updated_at.try(:to_s, :iso8601)
+    end
+
+    h[:email] = registrant.email
+    @disclosed << [:email, registrant.email]
+    h[:registrant_changed]          = registrant.updated_at.try(:to_s, :iso8601)
 
     h[:admin_contacts] = []
     domain.admin_contacts.each do |ac|
@@ -82,14 +90,14 @@ class WhoisRecord < ActiveRecord::Base
     h[:registrar_address] = domain.registrar.address
     h[:registrar_changed] = domain.registrar.updated_at.try(:to_s, :iso8601)
 
-    h[:nameservers]         = domain.nameservers.pluck(:hostname).uniq.select(&:present?)
+    h[:nameservers]         = domain.nameservers.hostnames.uniq.select(&:present?)
     h[:nameservers_changed] = domain.nameservers.pluck(:updated_at).max.try(:to_s, :iso8601)
 
     h[:dnssec_keys]    = domain.dnskeys.map{|key| "#{key.flags} #{key.protocol} #{key.alg} #{key.public_key}" }
     h[:dnssec_changed] = domain.dnskeys.pluck(:updated_at).max.try(:to_s, :iso8601) rescue nil
 
 
-    h[:disclosed] = @disclosed # later we can replace
+    h[:disclosed] = @disclosed
     h
   end
 

app/views/for_models/whois.erb

@@ -13,6 +13,8 @@ delete:     <%= json['delete'].to_s.tr('T',' ').sub('+', ' +') %>
 
 Registrant:
 name:    <%= json['registrant'] %>
+org id:  <%= json['registrant_reg_no'] %>
+country: <%= json['registrant_ident_country_code'] %>
 email:   Not Disclosed - Visit www.internet.ee for webbased WHOIS
 changed: <%= json['registrant_changed'].to_s.tr('T',' ').sub('+', ' +') %>
 

db/migrate/20160629114503_add_hash_to_legal_doc.rb

@@ -0,0 +1,6 @@
+class AddHashToLegalDoc < ActiveRecord::Migration
+  def change
+    add_column :legal_documents, :checksum, :string
+    add_index  :legal_documents, :checksum
+  end
+end

lib/tasks/legal_doc.rake

@@ -0,0 +1,35 @@
+namespace :legal_doc do
+
+  desc 'Legal documents duplicates fix'
+  task all: :environment do
+    Rake::Task['legal_doc:generate_hash'].invoke
+    Rake::Task['legal_doc:remove_duplicates'].invoke
+  end
+
+  desc 'Generate hash'
+  task generate_hash: :environment do
+    start = Time.zone.now.to_f
+    puts '-----> Generating unique hash for legal documents'
+    count = 0
+
+    LegalDocument.where(checksum: [nil, ""]).find_each do |x|
+      if File.exist?(x.path)
+        x.checksum = x.calc_checksum
+        x.save
+        count += 1
+      end
+
+    end
+    puts "-----> Hash generated for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds"
+  end
+
+
+  # Starting point is Domain legal docs
+  # then inside it checking the same domains and connected contacts
+  desc 'Remove duplicates'
+  task remove_duplicates: :environment do
+    LegalDocument.remove_duplicates
+  end
+
+end
+

spec/factories/contact.rb

@@ -12,5 +12,14 @@ FactoryGirl.define do
     ident_type 'priv'
     ident_country_code 'EE'
     registrar
+
+    factory :contact_private_entity do
+      ident_type 'priv'
+    end
+
+    factory :contact_legal_entity do
+      ident_type 'org'
+      ident '12345678' # valid reg no for .ee
+    end
   end
 end

spec/factories/registrant.rb

@@ -1,5 +1,8 @@
 FactoryGirl.define do
   factory :registrant, parent: :contact, class: Registrant do
     name 'test'
+
+    factory :registrant_private_entity, class: Registrant, parent: :contact_private_entity
+    factory :registrant_legal_entity, class: Registrant, parent: :contact_legal_entity
   end
 end

spec/models/contact_spec.rb

@@ -361,6 +361,8 @@ describe Contact, '.destroy_orphans' do
 end
 
 RSpec.describe Contact, db: false do
+  it { is_expected.to alias_attribute(:kind, :ident_type) }
+
   describe '::names' do
     before :example do
       expect(described_class).to receive(:pluck).with(:name).and_return('names')
@@ -463,4 +465,20 @@ RSpec.describe Contact, db: false do
       expect(address_removed).to be_truthy
     end
   end
+
+  describe '#reg_no' do
+    subject(:reg_no) { contact.reg_no }
+
+    context 'when contact is legal entity' do
+      let(:contact) { FactoryGirl.build_stubbed(:contact_legal_entity, ident: '1234') }
+
+      specify { expect(reg_no).to eq('1234') }
+    end
+
+    context 'when contact is private entity' do
+      let(:contact) { FactoryGirl.build_stubbed(:contact_private_entity, ident: '1234') }
+
+      specify { expect(reg_no).to be_nil }
+    end
+  end
 end

spec/models/legal_documents_spec.rb

@@ -0,0 +1,71 @@
+require 'rails_helper'
+
+describe LegalDocument do
+  context 'tasks' do
+    it 'make files uniq' do
+      Fabricate(:zonefile_setting, origin: 'ee')
+      Fabricate(:zonefile_setting, origin: 'pri.ee')
+      Fabricate(:zonefile_setting, origin: 'med.ee')
+      Fabricate(:zonefile_setting, origin: 'fie.ee')
+      Fabricate(:zonefile_setting, origin: 'com.ee')
+      LegalDocument.explicitly_write_file = true
+      PaperTrail.enabled = true
+
+      domain  = Fabricate(:domain)
+      domain2 = Fabricate(:domain)
+      legals = []
+      legals << original  = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << copy      = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << skipping_as_different_domain = domain2.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << skipping_as_different    = domain.legal_documents.create!(body: Base64.encode64('D' * 4.kilobytes))
+      legals << skipping_as_no_checksum  = domain.legal_documents.create!(checksum: nil, body: Base64.encode64('S' * 4.kilobytes))
+      legals << skipping_as_no_checksum2 = domain.legal_documents.create!(checksum: "",  body: Base64.encode64('S' * 4.kilobytes))
+      legals << registrant_copy = domain.registrant.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << registrant_skipping_as_different = domain.registrant.legal_documents.create!(body: Base64.encode64('Q' * 4.kilobytes))
+      legals << tech_copy = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << tech_skipping_as_different = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('W' * 4.kilobytes))
+      legals << admin_copy = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      legals << admin_skipping_as_different = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('E' * 4.kilobytes))
+      legals << new_second_tech_contact     = domain2.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes))
+      domain.tech_contacts << domain2.tech_contacts.first
+
+
+      # writing nesting to history
+      domain.update(updated_at: Time.now)
+      domain2.update(updated_at: Time.now)
+      domain.reload
+
+      skipping_as_no_checksum.update_columns(checksum: nil)
+      skipping_as_no_checksum2.update_columns(checksum: "")
+      skipping_as_no_checksum.reload
+      skipping_as_no_checksum2.reload
+      skipping_as_no_checksum.path.should_not == skipping_as_no_checksum2.path
+
+      skipping_as_no_checksum.checksum.should  == nil
+      skipping_as_no_checksum2.checksum.should == ""
+      original.checksum.should == copy.checksum
+      original.checksum.should_not == skipping_as_different.checksum
+      domain.tech_contacts.count.should == 2
+
+      LegalDocument.remove_duplicates
+      LegalDocument.remove_duplicates
+      LegalDocument.remove_duplicates
+      legals.each(&:reload)
+
+      skipping_as_no_checksum.path.should_not be(skipping_as_no_checksum2.path)
+      original.path.should_not == skipping_as_different.path
+      original.path.should_not == skipping_as_different_domain.path
+      original.path.should_not == registrant_skipping_as_different.path
+      original.path.should_not == tech_skipping_as_different.path
+      original.path.should_not == admin_skipping_as_different.path
+      original.path.should == copy.path
+      original.path.should == registrant_copy.path
+      original.path.should == tech_copy.path
+      original.path.should == admin_copy.path
+
+      original.path.should == new_second_tech_contact.path
+      skipping_as_different_domain.path.should_not == new_second_tech_contact.path
+    end
+  end
+
+end

spec/models/whois_record_spec.rb

@@ -0,0 +1,67 @@
+require 'rails_helper'
+
+RSpec.describe WhoisRecord do
+  describe '::generate_json', db: false do
+    let(:registrant) { FactoryGirl.build_stubbed(:registrant) }
+    let(:domain) { FactoryGirl.build_stubbed(:domain, registrant: registrant) }
+    let(:whois_record) { described_class.new }
+    subject(:generated_json) { whois_record.generate_json }
+
+    before do
+      allow(whois_record).to receive(:domain).and_return(domain)
+    end
+
+    it 'generates registrant kind' do
+      expect(registrant).to receive(:kind).and_return('test kind')
+      expect(generated_json[:registrant_kind]).to eq('test kind')
+    end
+
+    describe 'reg no' do
+      subject(:reg_no) { generated_json[:registrant_reg_no] }
+
+      before do
+        allow(registrant).to receive(:reg_no).and_return('test reg no')
+      end
+
+      context 'when registrant is legal entity' do
+        let(:registrant) { FactoryGirl.build_stubbed(:registrant_legal_entity) }
+
+        it 'is present' do
+          expect(reg_no).to eq('test reg no')
+        end
+      end
+
+      context 'when registrant is private entity' do
+        let(:registrant) { FactoryGirl.build_stubbed(:registrant_private_entity) }
+
+        it 'is absent' do
+          expect(reg_no).to be_nil
+        end
+      end
+    end
+
+    describe 'country code' do
+      subject(:country_code) { generated_json[:registrant_ident_country_code] }
+
+      before do
+        allow(registrant).to receive(:ident_country_code).and_return('test country code')
+      end
+
+      context 'when registrant is legal entity' do
+        let(:registrant) { FactoryGirl.build_stubbed(:registrant_legal_entity) }
+
+        it 'is present' do
+          expect(country_code).to eq('test country code')
+        end
+      end
+
+      context 'when registrant is private entity' do
+        let(:registrant) { FactoryGirl.build_stubbed(:registrant_private_entity) }
+
+        it 'is absent' do
+          expect(country_code).to be_nil
+        end
+      end
+    end
+  end
+end