diff --git a/CHANGELOG.md b/CHANGELOG.md index a58d3a268..a6517b4ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +22.12.2016 +* Return business registry code and country for 'org' type registrants in WHOIS and Rest-WHOIS + 16.12.2016 * Allow contact address processing to be configurable via admin * EPP XML schema namespace "urn:ietf:params:xml:ns:epp-1.0" replaced with "https://epp.tld.ee/schema/epp-ee-1.0.xsd" diff --git a/app/controllers/admin/registrars_controller.rb b/app/controllers/admin/registrars_controller.rb index be4b7d092..611687f9b 100644 --- a/app/controllers/admin/registrars_controller.rb +++ b/app/controllers/admin/registrars_controller.rb @@ -17,10 +17,15 @@ class Admin::RegistrarsController < AdminController def create @registrar = Registrar.new(registrar_params) - if @registrar.save + begin + @registrar.transaction do + @registrar.save! + @registrar.accounts.create!(account_type: Account::CASH, currency: 'EUR') + end + flash[:notice] = I18n.t('registrar_added') redirect_to [:admin, @registrar] - else + rescue ActiveRecord::RecordInvalid flash.now[:alert] = I18n.t('failed_to_add_registrar') render 'new' end diff --git a/app/models/contact.rb b/app/models/contact.rb index b50428bb1..99139e898 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -15,6 +15,7 @@ class Contact < ActiveRecord::Base has_paper_trail class_name: "ContactVersion", meta: { children: :children_log } attr_accessor :legal_document_id + alias_attribute :kind, :ident_type accepts_nested_attributes_for :legal_documents @@ -583,4 +584,9 @@ class Contact < ActiveRecord::Base self[attr_name.to_sym] = nil end end + + def reg_no + return if priv? + ident + end end diff --git a/app/models/domain_cron.rb b/app/models/domain_cron.rb index b4da056a2..dcd99328a 100644 --- a/app/models/domain_cron.rb +++ b/app/models/domain_cron.rb @@ -142,4 +142,9 @@ class DomainCron ) end + def self.delete_legal_doc_duplicates + Rake::Task['legal_doc:remove_duplicates'].reenable + Rake::Task['legal_doc:remove_duplicates'].invoke + end + end diff --git a/app/models/legal_document.rb b/app/models/legal_document.rb index b4bf3c96b..0c08958ae 100644 --- a/app/models/legal_document.rb +++ b/app/models/legal_document.rb @@ -1,4 +1,5 @@ class LegalDocument < ActiveRecord::Base + cattr_accessor :explicitly_write_file include EppErrors MIN_BODY_SIZE = (1.37 * 3.kilobytes).ceil @@ -16,7 +17,7 @@ class LegalDocument < ActiveRecord::Base validate :val_body_length, if: ->(file){ file.path.blank? && !Rails.env.staging?} before_create :add_creator - before_save :save_to_filesystem + before_save :save_to_filesystem, if: :body def epp_code_map { @@ -32,22 +33,82 @@ class LegalDocument < ActiveRecord::Base def save_to_filesystem - loop do - rand = SecureRandom.random_number.to_s.last(4) - next if rand.to_i == 0 || rand.length < 4 + binary = Base64.decode64(body) + digest = Digest::SHA1.new.update(binary).to_s - dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}" - FileUtils.mkdir_p(dir, mode: 0775) - self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}" - break unless File.file?(path) + loop do + rand = SecureRandom.random_number.to_s.last(4) + next if rand.to_i == 0 || rand.length < 4 + dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}" + FileUtils.mkdir_p(dir, mode: 0775) + self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}" + break unless File.file?(path) end - File.open(path, 'wb') { |f| f.write(Base64.decode64(body)) } unless Rails.env.test? + File.open(path, 'wb') { |f| f.write(binary) } if !Rails.env.test? || self.class.explicitly_write_file self.path = path + self.checksum = digest + end + + def calc_checksum + digest = Digest::SHA1.new + digest.update File.binread(path) + digest.to_s end def add_creator self.creator_str = ::PaperTrail.whodunnit true end + + + def self.remove_duplicates + start = Time.zone.now.to_f + puts '-----> Removing legal documents duplicates' + count = 0 + modified = Array.new + + LegalDocument.where(documentable_type: "Domain").where.not(checksum: [nil, ""]).find_each do |orig_legal| + next if modified.include?(orig_legal.checksum) + next if !File.exist?(orig_legal.path) + modified.push(orig_legal.checksum) + + LegalDocument.where(documentable_type: "Domain", documentable_id: orig_legal.documentable_id). + where(checksum: orig_legal.checksum). + where.not(id: orig_legal.id).where.not(path: orig_legal.path).each do |new_legal| + unless modified.include?(orig_legal.id) + File.delete(new_legal.path) if File.exist?(new_legal.path) + new_legal.update(path: orig_legal.path) + count += 1 + puts "File #{new_legal.path} has been removed by Domain #{new_legal.documentable_id}. Document id: #{new_legal.id}" + end + end + + contact_ids = DomainVersion.where(item_id: orig_legal.documentable_id).distinct. + pluck("object->>'registrant_id'", "object_changes->>'registrant_id'", + "children->>'tech_contacts'", "children->>'admin_contacts'", + "tech_contact_ids", "admin_contact_ids").flatten.uniq + contact_ids = contact_ids.map{|id| + case id + when Hash + id["id"] + when String + JSON.parse(id) rescue id.to_i + else + id + end + }.flatten.compact.uniq + LegalDocument.where(documentable_type: "Contact", documentable_id: contact_ids). + where(checksum: orig_legal.checksum).where.not(path: orig_legal.path).each do |new_legal| + unless modified.include?(orig_legal.id) + File.delete(new_legal.path) if File.exist?(new_legal.path) + new_legal.update(path: orig_legal.path) + count += 1 + puts "File #{new_legal.path} has been removed by Contact #{new_legal.documentable_id}. Document id: #{new_legal.id}" + end + end + end + puts "-----> Duplicates fixed for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds" + + end end diff --git a/app/models/registrar.rb b/app/models/registrar.rb index 003956f88..162aef292 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -57,11 +57,6 @@ class Registrar < ActiveRecord::Base RegenerateRegistrarWhoisesJob.enqueue id end - after_create :create_cash_account - def create_cash_account - accounts.create(account_type: Account::CASH, currency: 'EUR') - end - class << self def search_by_query(query) res = search(name_or_reg_no_cont: query).result diff --git a/app/models/whois_record.rb b/app/models/whois_record.rb index c16e5ce73..e5de62ac0 100644 --- a/app/models/whois_record.rb +++ b/app/models/whois_record.rb @@ -41,6 +41,8 @@ class WhoisRecord < ActiveRecord::Base 'ok' => 'ok (paid and in zone)' } + registrant = domain.registrant + @disclosed = [] h[:name] = domain.name h[:status] = domain.statuses.map { |x| status_map[x] || x } @@ -50,11 +52,17 @@ class WhoisRecord < ActiveRecord::Base h[:outzone] = domain.outzone_at.try(:to_date).try(:to_s) h[:delete] = [domain.delete_at, domain.force_delete_at].compact.min.try(:to_date).try(:to_s) + h[:registrant] = registrant.name + h[:registrant_kind] = registrant.kind - h[:registrant] = domain.registrant.name - h[:email] = domain.registrant.email - @disclosed << [:email, domain.registrant.email] - h[:registrant_changed] = domain.registrant.updated_at.try(:to_s, :iso8601) + if registrant.org? + h[:registrant_reg_no] = registrant.reg_no + h[:registrant_ident_country_code] = registrant.ident_country_code + end + + h[:email] = registrant.email + @disclosed << [:email, registrant.email] + h[:registrant_changed] = registrant.updated_at.try(:to_s, :iso8601) h[:admin_contacts] = [] domain.admin_contacts.each do |ac| @@ -82,14 +90,14 @@ class WhoisRecord < ActiveRecord::Base h[:registrar_address] = domain.registrar.address h[:registrar_changed] = domain.registrar.updated_at.try(:to_s, :iso8601) - h[:nameservers] = domain.nameservers.pluck(:hostname).uniq.select(&:present?) + h[:nameservers] = domain.nameservers.hostnames.uniq.select(&:present?) h[:nameservers_changed] = domain.nameservers.pluck(:updated_at).max.try(:to_s, :iso8601) h[:dnssec_keys] = domain.dnskeys.map{|key| "#{key.flags} #{key.protocol} #{key.alg} #{key.public_key}" } h[:dnssec_changed] = domain.dnskeys.pluck(:updated_at).max.try(:to_s, :iso8601) rescue nil - h[:disclosed] = @disclosed # later we can replace + h[:disclosed] = @disclosed h end diff --git a/app/views/for_models/whois.erb b/app/views/for_models/whois.erb index 030f10d51..ba218e06f 100644 --- a/app/views/for_models/whois.erb +++ b/app/views/for_models/whois.erb @@ -13,6 +13,8 @@ delete: <%= json['delete'].to_s.tr('T',' ').sub('+', ' +') %> Registrant: name: <%= json['registrant'] %> +org id: <%= json['registrant_reg_no'] %> +country: <%= json['registrant_ident_country_code'] %> email: Not Disclosed - Visit www.internet.ee for webbased WHOIS changed: <%= json['registrant_changed'].to_s.tr('T',' ').sub('+', ' +') %> diff --git a/db/migrate/20160629114503_add_hash_to_legal_doc.rb b/db/migrate/20160629114503_add_hash_to_legal_doc.rb new file mode 100644 index 000000000..8ea2f182d --- /dev/null +++ b/db/migrate/20160629114503_add_hash_to_legal_doc.rb @@ -0,0 +1,6 @@ +class AddHashToLegalDoc < ActiveRecord::Migration + def change + add_column :legal_documents, :checksum, :string + add_index :legal_documents, :checksum + end +end diff --git a/lib/tasks/legal_doc.rake b/lib/tasks/legal_doc.rake new file mode 100644 index 000000000..ad7df0fd2 --- /dev/null +++ b/lib/tasks/legal_doc.rake @@ -0,0 +1,35 @@ +namespace :legal_doc do + + desc 'Legal documents duplicates fix' + task all: :environment do + Rake::Task['legal_doc:generate_hash'].invoke + Rake::Task['legal_doc:remove_duplicates'].invoke + end + + desc 'Generate hash' + task generate_hash: :environment do + start = Time.zone.now.to_f + puts '-----> Generating unique hash for legal documents' + count = 0 + + LegalDocument.where(checksum: [nil, ""]).find_each do |x| + if File.exist?(x.path) + x.checksum = x.calc_checksum + x.save + count += 1 + end + + end + puts "-----> Hash generated for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds" + end + + + # Starting point is Domain legal docs + # then inside it checking the same domains and connected contacts + desc 'Remove duplicates' + task remove_duplicates: :environment do + LegalDocument.remove_duplicates + end + +end + diff --git a/spec/factories/contact.rb b/spec/factories/contact.rb index 5218da399..7122babd5 100644 --- a/spec/factories/contact.rb +++ b/spec/factories/contact.rb @@ -12,5 +12,14 @@ FactoryGirl.define do ident_type 'priv' ident_country_code 'EE' registrar + + factory :contact_private_entity do + ident_type 'priv' + end + + factory :contact_legal_entity do + ident_type 'org' + ident '12345678' # valid reg no for .ee + end end end diff --git a/spec/factories/registrant.rb b/spec/factories/registrant.rb index c4846fcd4..f25942829 100644 --- a/spec/factories/registrant.rb +++ b/spec/factories/registrant.rb @@ -1,5 +1,8 @@ FactoryGirl.define do factory :registrant, parent: :contact, class: Registrant do name 'test' + + factory :registrant_private_entity, class: Registrant, parent: :contact_private_entity + factory :registrant_legal_entity, class: Registrant, parent: :contact_legal_entity end end diff --git a/spec/models/contact_spec.rb b/spec/models/contact_spec.rb index 3952a1d0e..df4d8cfd3 100644 --- a/spec/models/contact_spec.rb +++ b/spec/models/contact_spec.rb @@ -361,6 +361,8 @@ describe Contact, '.destroy_orphans' do end RSpec.describe Contact, db: false do + it { is_expected.to alias_attribute(:kind, :ident_type) } + describe '::names' do before :example do expect(described_class).to receive(:pluck).with(:name).and_return('names') @@ -463,4 +465,20 @@ RSpec.describe Contact, db: false do expect(address_removed).to be_truthy end end + + describe '#reg_no' do + subject(:reg_no) { contact.reg_no } + + context 'when contact is legal entity' do + let(:contact) { FactoryGirl.build_stubbed(:contact_legal_entity, ident: '1234') } + + specify { expect(reg_no).to eq('1234') } + end + + context 'when contact is private entity' do + let(:contact) { FactoryGirl.build_stubbed(:contact_private_entity, ident: '1234') } + + specify { expect(reg_no).to be_nil } + end + end end diff --git a/spec/models/legal_documents_spec.rb b/spec/models/legal_documents_spec.rb new file mode 100644 index 000000000..e411c923d --- /dev/null +++ b/spec/models/legal_documents_spec.rb @@ -0,0 +1,71 @@ +require 'rails_helper' + +describe LegalDocument do + context 'tasks' do + it 'make files uniq' do + Fabricate(:zonefile_setting, origin: 'ee') + Fabricate(:zonefile_setting, origin: 'pri.ee') + Fabricate(:zonefile_setting, origin: 'med.ee') + Fabricate(:zonefile_setting, origin: 'fie.ee') + Fabricate(:zonefile_setting, origin: 'com.ee') + LegalDocument.explicitly_write_file = true + PaperTrail.enabled = true + + domain = Fabricate(:domain) + domain2 = Fabricate(:domain) + legals = [] + legals << original = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << copy = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_different_domain = domain2.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_different = domain.legal_documents.create!(body: Base64.encode64('D' * 4.kilobytes)) + legals << skipping_as_no_checksum = domain.legal_documents.create!(checksum: nil, body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_no_checksum2 = domain.legal_documents.create!(checksum: "", body: Base64.encode64('S' * 4.kilobytes)) + legals << registrant_copy = domain.registrant.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << registrant_skipping_as_different = domain.registrant.legal_documents.create!(body: Base64.encode64('Q' * 4.kilobytes)) + legals << tech_copy = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << tech_skipping_as_different = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('W' * 4.kilobytes)) + legals << admin_copy = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << admin_skipping_as_different = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('E' * 4.kilobytes)) + legals << new_second_tech_contact = domain2.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + domain.tech_contacts << domain2.tech_contacts.first + + + # writing nesting to history + domain.update(updated_at: Time.now) + domain2.update(updated_at: Time.now) + domain.reload + + skipping_as_no_checksum.update_columns(checksum: nil) + skipping_as_no_checksum2.update_columns(checksum: "") + skipping_as_no_checksum.reload + skipping_as_no_checksum2.reload + skipping_as_no_checksum.path.should_not == skipping_as_no_checksum2.path + + skipping_as_no_checksum.checksum.should == nil + skipping_as_no_checksum2.checksum.should == "" + original.checksum.should == copy.checksum + original.checksum.should_not == skipping_as_different.checksum + domain.tech_contacts.count.should == 2 + + LegalDocument.remove_duplicates + LegalDocument.remove_duplicates + LegalDocument.remove_duplicates + legals.each(&:reload) + + skipping_as_no_checksum.path.should_not be(skipping_as_no_checksum2.path) + original.path.should_not == skipping_as_different.path + original.path.should_not == skipping_as_different_domain.path + original.path.should_not == registrant_skipping_as_different.path + original.path.should_not == tech_skipping_as_different.path + original.path.should_not == admin_skipping_as_different.path + original.path.should == copy.path + original.path.should == registrant_copy.path + original.path.should == tech_copy.path + original.path.should == admin_copy.path + + original.path.should == new_second_tech_contact.path + skipping_as_different_domain.path.should_not == new_second_tech_contact.path + end + end + +end \ No newline at end of file diff --git a/spec/models/whois_record_spec.rb b/spec/models/whois_record_spec.rb new file mode 100644 index 000000000..0f404a4de --- /dev/null +++ b/spec/models/whois_record_spec.rb @@ -0,0 +1,67 @@ +require 'rails_helper' + +RSpec.describe WhoisRecord do + describe '::generate_json', db: false do + let(:registrant) { FactoryGirl.build_stubbed(:registrant) } + let(:domain) { FactoryGirl.build_stubbed(:domain, registrant: registrant) } + let(:whois_record) { described_class.new } + subject(:generated_json) { whois_record.generate_json } + + before do + allow(whois_record).to receive(:domain).and_return(domain) + end + + it 'generates registrant kind' do + expect(registrant).to receive(:kind).and_return('test kind') + expect(generated_json[:registrant_kind]).to eq('test kind') + end + + describe 'reg no' do + subject(:reg_no) { generated_json[:registrant_reg_no] } + + before do + allow(registrant).to receive(:reg_no).and_return('test reg no') + end + + context 'when registrant is legal entity' do + let(:registrant) { FactoryGirl.build_stubbed(:registrant_legal_entity) } + + it 'is present' do + expect(reg_no).to eq('test reg no') + end + end + + context 'when registrant is private entity' do + let(:registrant) { FactoryGirl.build_stubbed(:registrant_private_entity) } + + it 'is absent' do + expect(reg_no).to be_nil + end + end + end + + describe 'country code' do + subject(:country_code) { generated_json[:registrant_ident_country_code] } + + before do + allow(registrant).to receive(:ident_country_code).and_return('test country code') + end + + context 'when registrant is legal entity' do + let(:registrant) { FactoryGirl.build_stubbed(:registrant_legal_entity) } + + it 'is present' do + expect(country_code).to eq('test country code') + end + end + + context 'when registrant is private entity' do + let(:registrant) { FactoryGirl.build_stubbed(:registrant_private_entity) } + + it 'is absent' do + expect(country_code).to be_nil + end + end + end + end +end