From 3d0d98d9b15496ee464313b80bde5214fc893d7a Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 10:39:52 +0300 Subject: [PATCH 01/11] Refactored whois from domain into whois body --- app/models/domain.rb | 51 +-------- app/models/domain_status.rb | 9 ++ app/models/whois.rb | 2 - app/models/whois/body.rb | 107 ++++++++++++++++++ .../20150421134820_add_whosi_json_body.rb | 5 + db/schema.rb | 1 + spec/models/domain_spec.rb | 80 ++++++++++++- 7 files changed, 204 insertions(+), 51 deletions(-) delete mode 100644 app/models/whois.rb create mode 100644 app/models/whois/body.rb create mode 100644 db/migrate/20150421134820_add_whosi_json_body.rb diff --git a/app/models/domain.rb b/app/models/domain.rb index 3b71185eb..01ed98484 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -238,55 +238,10 @@ class Domain < ActiveRecord::Base log end - # rubocop:disable Metrics/MethodLength def update_whois_body - self.whois_body = <<-EOS -Estonia .ee Top Level Domain WHOIS server - - domain: #{name} - registrant: #{registrant.name} - status: #{domain_statuses.map(&:value).join(', ')} - registered: #{registered_at and registered_at.to_s(:db)} - changed: #{updated_at and updated_at.to_s(:db)} - expire: #{valid_to and valid_to.to_s(:db)} - outzone: - delete: - - #{contacts_body} - - nsset: - nserver: - - registrar: #{registrar} - phone: #{registrar.phone} - address: #{registrar.address} - created: #{registrar.created_at.to_s(:db)} - changed: #{registrar.updated_at.to_s(:db)} - -Estonia .ee Top Level Domain WHOIS server -More information at http://internet.ee - EOS - end - # rubocop:enable Metrics/MethodLength - - def contacts_body - out = '' - admin_contacts.each do |c| - out << 'Admin contact:' - out << "name: #{c.name}" - out << "email: #{c.email}" - out << "registrar: #{c.registrar}" - out << "created: #{c.created_at.to_s(:db)}" - end - - tech_contacts.each do |c| - out << 'Tech contact:' - out << "name: #{c.name}" - out << "email: #{c.email}" - out << "registrar: #{c.registrar}" - out << "created: #{c.created_at.to_s(:db)}" - end - out + whois = Whois::Body.new(self) + self.whois_json = whois.whois_json + self.whois_body = whois.whois_body end def update_whois_server diff --git a/app/models/domain_status.rb b/app/models/domain_status.rb index 83bbf1308..5a435ecc4 100644 --- a/app/models/domain_status.rb +++ b/app/models/domain_status.rb @@ -109,6 +109,15 @@ class DomainStatus < ActiveRecord::Base CLIENT_STATUSES.include?(value) end + def human_value + case value + when 'ok' + 'ok (paid and in zone)' + else + value + end + end + class << self def statuses_for_client CLIENT_STATUSES.map { |x| x.sub('client', '') } diff --git a/app/models/whois.rb b/app/models/whois.rb deleted file mode 100644 index 1364d5099..000000000 --- a/app/models/whois.rb +++ /dev/null @@ -1,2 +0,0 @@ -module Whois -end diff --git a/app/models/whois/body.rb b/app/models/whois/body.rb new file mode 100644 index 000000000..2483b6c60 --- /dev/null +++ b/app/models/whois/body.rb @@ -0,0 +1,107 @@ +class Whois::Body + # rubocop:disable Metrics/MethodLength + def h + @h ||= HashWithIndifferentAccess.new + end + + def initialize(domain) + h[:name] = domain.name + h[:registrant] = domain.registrant.name + h[:status] = domain.domain_statuses.map(&:human_value).join(', ') + h[:registered] = domain.registered_at and domain.registered_at.to_s(:db) + h[:updated_at] = domain.updated_at and domain.updated_at.to_s(:db) + h[:valid_to] = domain.valid_to and domain.valid_to.to_s(:db) + + h[:registrar] = domain.registrar.name + h[:registrar_phone] = domain.registrar.phone + h[:registrar_address] = domain.registrar.address + h[:registrar_update_at] = domain.registrar.updated_at.to_s(:db) + h[:admin_contacts] = [] + domain.admin_contacts.each do |ac| + h[:admin_contacts] << { + name: ac.name, + email: ac.email, + registrar: ac.registrar.name, + created_at: ac.created_at.to_s(:db) + } + end + h[:tech_contacts] = [] + domain.tech_contacts.each do |tc| + h[:tech_contacts] << { + name: tc.name, + email: tc.email, + registrar: tc.registrar.name, + created_at: tc.created_at.to_s(:db) + } + end + h[:nameservers] = [] + domain.nameservers.each do |ns| + h[:nameservers] << { + hostname: ns.hostname, + updated_at: ns.updated_at.to_s(:db) + } + end + end + + def whois_json + h + end + + def whois_body + <<-EOS +Estonia .ee Top Level Domain WHOIS server + +Domain: + name: #{h[:name]} + registrant: #{h[:registrant]} + status: #{h[:status]} + registered: #{h[:registered]} + changed: #{h[:updated_at]} + expire: #{h[:valid_to]} + outzone: + delete: +#{contacts_body(h[:admin_contacts], h[:tech_contacts])} +Registrar: + name: #{h[:registrar]} + phone: #{h[:registrar_phone]} + address: #{h[:registrar_address]} + changed: #{h[:registrar_update_at]} +#{nameservers_body(h[:nameservers])} +Estonia .ee Top Level Domain WHOIS server +More information at http://internet.ee + EOS + end + # rubocop:enable Metrics/MethodLength + + def contacts_body(admins, techs) + out = '' + out << (admins.size > 1 ? "\nAdministrative contacts" : "\nAdministrative contact") + admins.each do |c| + out << "\n name: #{c[:name]}" + out << "\n e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n registrar: #{c[:registrar]}" + out << "\n created: #{c[:created_at]}" + out << "\n" + end + + out << (techs.size > 1 ? "\nTechnical contacts" : "\nTechnical contact:") + techs.each do |c| + out << "\n name: #{c[:name]}" + out << "\n e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n registrar: #{c[:registrar]}" + out << "\n created: #{c[:created_at]}" + out << "\n" + end + out + end + + def nameservers_body(nservers) + out = "\nName servers:" + nservers.each do |ns| + out << "\n nserver: #{ns[:hostname]}" + out << "\n changed: #{ns[:updated_at]}" + out << "\n" + end + out + end +end diff --git a/db/migrate/20150421134820_add_whosi_json_body.rb b/db/migrate/20150421134820_add_whosi_json_body.rb new file mode 100644 index 000000000..c1bf00075 --- /dev/null +++ b/db/migrate/20150421134820_add_whosi_json_body.rb @@ -0,0 +1,5 @@ +class AddWhosiJsonBody < ActiveRecord::Migration + def change + add_column :domains, :whois_json, :json + end +end diff --git a/db/schema.rb b/db/schema.rb index 755d6eecc..21b95105b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -298,6 +298,7 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.integer "legacy_id" t.integer "legacy_registrar_id" t.integer "legacy_registrant_id" + t.json "whois_json" end add_index "domains", ["registrant_id"], name: "index_domains_on_registrant_id", using: :btree diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index bb8881ff1..90f1a156e 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -38,6 +38,9 @@ describe Domain do @domain.whois_body.should == nil end + it 'should not have whois json' do + @domain.whois_json.should == nil + end end context 'with valid attributes' do @@ -74,10 +77,85 @@ describe Domain do domain.errors.full_messages.should match_array(["Admin domain contacts is invalid"]) end - it 'should not have whois_body present by default' do + it 'should have whois body by default' do @domain.whois_body.present?.should == true end + it 'should have whois json by default' do + @domain.whois_json.present?.should == true + end + + it 'should have whois_body present by default' do + @domain.name = 'yeah.ee' + @domain.updated_at = Time.zone.parse('2020.02.02 02:00') + @domain.registered_at = Time.zone.parse('2000.01.01 9:00') + @domain.valid_to = Time.zone.parse('2016.04.21 0:00') + registrar = Fabricate(:registrar, + name: 'First Registrar Ltd', + created_at: Time.zone.parse('1995.01.01'), + updated_at: Time.zone.parse('1996.01.01')) + @domain.registrant = Fabricate(:contact, + name: 'Jarren Jakubowski0', + created_at: Time.zone.parse('2005.01.01')) + @domain.admin_contacts = [Fabricate(:contact, + name: 'First Admin', + registrar: registrar, + created_at: Time.zone.parse('2016.01.01'))] + @domain.tech_contacts = [Fabricate(:contact, + name: 'First Tech', + registrar: registrar, + created_at: Time.zone.parse('2016.01.01'))] + @domain.registrar = registrar + ns1 = Fabricate(:nameserver, hostname: 'test.ee') + ns1.updated_at = Time.zone.parse('1980.01.01') + ns2 = Fabricate(:nameserver, hostname: 'test1.ee') + ns2.updated_at = Time.zone.parse('1970.01.01') + @domain.nameservers = [ns1, ns2] + + @domain.update_whois_body + @domain.whois_body.should == <<-EOS +Estonia .ee Top Level Domain WHOIS server + +Domain: + name: yeah.ee + registrant: Jarren Jakubowski0 + status: ok (paid and in zone) + registered: 2000-01-01 09:00:00 UTC + changed: 2020-02-02 02:00:00 UTC + expire: 2016-04-21 00:00:00 UTC + outzone: + delete: + +Administrative contact + name: First Admin + e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS + registrar: First Registrar Ltd + created: 2016-01-01 00:00:00 + +Technical contact: + name: First Tech + e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS + registrar: First Registrar Ltd + created: 2016-01-01 00:00:00 + +Registrar: + name: First Registrar Ltd + phone: + address: Street 999, Town, County, Postal + changed: 1996-01-01 00:00:00 + +Name servers: + nserver: test.ee + changed: 1980-01-01 00:00:00 + + nserver: test1.ee + changed: 1970-01-01 00:00:00 + +Estonia .ee Top Level Domain WHOIS server +More information at http://internet.ee + EOS + end + context 'with versioning' do it 'should not have one version' do with_versioning do From 8dae9d9deb9cc72c76c9a7c95a6593268272bd88 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 11:51:33 +0300 Subject: [PATCH 02/11] Updated whois tasks and added whois_json --- app/models/domain.rb | 12 +++++++--- db/whois_schema.rb | 1 + lib/tasks/db.rake | 4 ++-- lib/tasks/whois.rake | 56 +++++++++++++++++++++++++------------------- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/app/models/domain.rb b/app/models/domain.rb index 01ed98484..4855ad41a 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -226,6 +226,9 @@ class Domain < ActiveRecord::Base domain_statuses.find_by(value: DomainStatus::OK).try(:destroy) end + # otherwise domain_statuses are in old state for domain object + domain_statuses.reload + # contacts.includes(:address).each(&:manage_statuses) end @@ -234,14 +237,17 @@ class Domain < ActiveRecord::Base log[:admin_contacts] = admin_contacts.map(&:attributes) log[:tech_contacts] = tech_contacts.map(&:attributes) log[:nameservers] = nameservers.map(&:attributes) - log[:registrant] = [registrant.try(:attributes)] + log[:registrant] = [registrant.try(:attributes)] log end def update_whois_body whois = Whois::Body.new(self) - self.whois_json = whois.whois_json - self.whois_body = whois.whois_body + # validations, callbacks and updated_at are skipped + update_columns( + whois_json: whois.whois_json, + whois_body: whois.whois_body + ) end def update_whois_server diff --git a/db/whois_schema.rb b/db/whois_schema.rb index 0290bd178..0ddeb58b3 100644 --- a/db/whois_schema.rb +++ b/db/whois_schema.rb @@ -19,6 +19,7 @@ ActiveRecord::Schema.define(version: 20150113113236) do create_table "domains", force: :cascade do |t| t.string "name" t.text "whois_body" + t.json "whois_json" t.datetime "created_at", null: false t.datetime "updated_at", null: false end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index aa06aa273..cd022bcb1 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -33,7 +33,7 @@ namespace :db do task create: [:environment, :load_config] do databases.each do |name| begin - puts "\n---------------------------- Create #{name} ----------------------------------------\n" + puts "\n------------------------ Create #{name} ---------------------------------------\n" ActiveRecord::Base.clear_all_connections! conf = ActiveRecord::Base.configurations @@ -75,7 +75,7 @@ namespace :db do task load: [:environment, :load_config] do databases.each do |name| begin - puts "\n---------------------------- #{name} schema loaded ----------------------------------------\n" + puts "\n------------------------ #{name} schema loading -----------------------------\n" ActiveRecord::Base.clear_all_connections! ActiveRecord::Base.establish_connection(name.to_sym) if ActiveRecord::Base.connection.table_exists?('schema_migrations') diff --git a/lib/tasks/whois.rake b/lib/tasks/whois.rake index 29d15fcb8..359db50da 100644 --- a/lib/tasks/whois.rake +++ b/lib/tasks/whois.rake @@ -1,33 +1,41 @@ namespace :whois do - desc 'Delete whois database data and import all from Registry (fast)' - task reset: :environment do + desc 'Regenerate whois_body and whois_json at Registry master database (slow)' + task regenerate: :environment do start = Time.zone.now.to_f - print "-----> Reset whois database and sync..." - domains = Domain.pluck(:name, :whois_body) + print "-----> Regenerate whois_body and whois_json at Registry master database..." + Domain.included.find_each(batch_size: 50000).with_index do |d, index| + d.update_whois_body + print '.' if index % 100 == 0 + end + puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" + end + + desc 'Delete whois database data and sync with Registry master database (fast)' + task export: :environment do + start = Time.zone.now.to_f + print "-----> Delete whois database data and sync with Registry master database..." + domains = Domain.pluck(:name, :whois_body, :whois_json) Whois::Domain.delete_all - Whois::Domain.import([:name, :whois_body], domains) + Whois::Domain.import([:name, :whois_body, :whois_json], domains) puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end - desc 'Sync whois database without reset (slow)' - task sync: :environment do - start = Time.zone.now.to_f - print "-----> Sync whois database..." - Domain.select(:id, :name, :whois_body).find_each(batch_size: 100000).with_index do |d, index| - d.update_whois_server - print '.' if index % 100 == 0 + namespace :schema do + desc 'Load whois schema' + task load: [:environment] do + whois_db = "whois_#{Rails.env}" + begin + puts "\n------------------------ #{whois_db} schema loading ------------------------------\n" + ActiveRecord::Base.clear_all_connections! + ActiveRecord::Base.establish_connection(whois_db.to_sym) + if ActiveRecord::Base.connection.table_exists?('schema_migrations') + puts 'Found tables, skip schema load!' + else + load("#{Rails.root}/db/#{schema_file(whois_db)}") + end + rescue => e + puts "\n#{e}" + end end - puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" - end - - desc 'Regenerate whois_body at Registry master database (slow)' - task generate: :environment do - start = Time.zone.now.to_f - print "-----> Update Registry records..." - Domain.included.find_each(batch_size: 100000).with_index do |d, index| - d.update_columns(whois_body: d.update_whois_body) - print '.' if index % 100 == 0 - end - puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end end From 510bde7d53cc8817cc3e762dcf8da4c575273097 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 11:56:03 +0300 Subject: [PATCH 03/11] Added changelog info about rest whois --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index add9ce179..798980aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,19 @@ 24.04.2015 - * Update zonefile procedure +* Update zonefile procedure 23.04.2015 - - * Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup` + +* Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup` 22.04.2015 * Configure smtp (see application-example.yml) +22.04.2015 + +* Whois database schema updated. Please reset whois database and run `rake whois:schema:load` + 21.04.2015 * Install packages for wkhtmltopdf (see readme) From 82fd2db963375c67d9e2fa099459dc27f474ea38 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 13:33:02 +0300 Subject: [PATCH 04/11] Update seeds to work with new roles requirements --- db/seeds.rb | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index 03dc6c18f..892cc9dfe 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -39,29 +39,35 @@ ApiUser.where( registrar: registrar2 ).first_or_create! -AdminUser.where( +admin1 = { username: 'user1', password: 'test1', email: 'user1@example.ee', identity_code: '37810013855', country_code: 'EE' -).first_or_create! - -AdminUser.where( +} +admin2 = { username: 'user2', password: 'test2', email: 'user2@example.ee', identity_code: '37810010085', country_code: 'EE' -).first_or_create! - -AdminUser.where( +} +admin3 = { username: 'user3', password: 'test3', email: 'user3@example.ee', identity_code: '37810010727', country_code: 'EE' -).first_or_create! +} + +[admin1, admin2, admin3].each do |at| + admin = AdminUser.where(at) + next if admin.present? + admin = AdminUser.new(at) + admin.roles = ['admin'] + admin.save +end ZonefileSetting.where({ origin: 'ee', @@ -85,8 +91,6 @@ ZonefileSetting.where({ master_nameserver: 'ns.tld.ee' }).first_or_create! -AdminUser.update_all(roles: ['admin']) - Registrar.where( name: 'EIS', reg_no: '90010019', From db81a9e7bc38d14e1dbdd8ebf93c79a13cb50da4 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 16:41:52 +0300 Subject: [PATCH 05/11] WhoisBody refactored to has_one --- app/models/domain.rb | 18 ++++++------- app/models/{whois/body.rb => whois_body.rb} | 25 +++++++++++++------ ...150422092514_add_whois_body_to_registry.rb | 15 +++++++++++ db/schema.rb | 17 +++++++++++-- lib/tasks/whois.rake | 4 +-- spec/models/domain_spec.rb | 10 +++----- 6 files changed, 62 insertions(+), 27 deletions(-) rename app/models/{whois/body.rb => whois_body.rb} (88%) create mode 100644 db/migrate/20150422092514_add_whois_body_to_registry.rb diff --git a/app/models/domain.rb b/app/models/domain.rb index 4855ad41a..ae8e24540 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -33,6 +33,7 @@ class Domain < ActiveRecord::Base has_many :dnskeys, dependent: :destroy has_many :keyrelays + has_one :whois_body, dependent: :destroy accepts_nested_attributes_for :dnskeys, allow_destroy: true @@ -124,6 +125,7 @@ class Domain < ActiveRecord::Base includes( :registrar, :nameservers, + :whois_body, { tech_contacts: :registrar }, { admin_contacts: :registrar } ) @@ -242,17 +244,15 @@ class Domain < ActiveRecord::Base end def update_whois_body - whois = Whois::Body.new(self) - # validations, callbacks and updated_at are skipped - update_columns( - whois_json: whois.whois_json, - whois_body: whois.whois_body - ) + self.whois_body = WhoisBody.create if whois_body.blank? + whois_body.update end def update_whois_server - wd = Whois::Domain.find_or_initialize_by(name: name) - wd.whois_body = whois_body - wd.save + if whois_body.present? + whois_body.update_whois_server + else + logger.info "NO WHOIS BODY for domain: #{name}" + end end end diff --git a/app/models/whois/body.rb b/app/models/whois_body.rb similarity index 88% rename from app/models/whois/body.rb rename to app/models/whois_body.rb index 2483b6c60..29e1917b9 100644 --- a/app/models/whois/body.rb +++ b/app/models/whois_body.rb @@ -1,10 +1,20 @@ -class Whois::Body +class WhoisBody < ActiveRecord::Base + belongs_to :domain + + def update_whois_server + return logger.info "NO WHOIS NAME for whois_body id: #{id}" if name.blank? + wd = Whois::Domain.find_or_initialize_by(name: name) + wd.whois_body = whois_body + wd.whois_json = whois_json + wd.save + end + # rubocop:disable Metrics/MethodLength def h @h ||= HashWithIndifferentAccess.new end - def initialize(domain) + def update h[:name] = domain.name h[:registrant] = domain.registrant.name h[:status] = domain.domain_statuses.map(&:human_value).join(', ') @@ -41,13 +51,14 @@ class Whois::Body updated_at: ns.updated_at.to_s(:db) } end + + self.name = h[:name] + self.whois_body = body + self.whois_json = h + save end - def whois_json - h - end - - def whois_body + def body <<-EOS Estonia .ee Top Level Domain WHOIS server diff --git a/db/migrate/20150422092514_add_whois_body_to_registry.rb b/db/migrate/20150422092514_add_whois_body_to_registry.rb new file mode 100644 index 000000000..6791bbcf9 --- /dev/null +++ b/db/migrate/20150422092514_add_whois_body_to_registry.rb @@ -0,0 +1,15 @@ +class AddWhoisBodyToRegistry < ActiveRecord::Migration + def change + create_table :whois_bodies, force: :cascade do |t| + t.integer :domain_id + t.string :name + t.text :whois_body + t.json :whois_json + t.datetime :created_at, null: false + t.datetime :updated_at, null: false + end + add_index :whois_bodies, :domain_id + remove_column :domains, :whois_body, :text + remove_column :domains, :whois_json, :json + end +end diff --git a/db/schema.rb b/db/schema.rb index 21b95105b..3ecb2783b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -294,13 +294,15 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.string "period_unit", limit: 1 t.string "creator_str" t.string "updator_str" - t.text "whois_body" t.integer "legacy_id" t.integer "legacy_registrar_id" t.integer "legacy_registrant_id" - t.json "whois_json" + t.datetime "outzone_at" + t.datetime "delete_at" end + add_index "domains", ["delete_at"], name: "index_domains_on_delete_at", using: :btree + add_index "domains", ["outzone_at"], name: "index_domains_on_outzone_at", using: :btree add_index "domains", ["registrant_id"], name: "index_domains_on_registrant_id", using: :btree add_index "domains", ["registrar_id"], name: "index_domains_on_registrar_id", using: :btree @@ -908,6 +910,17 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.text "depricated_table_but_somehow_paper_trail_tests_fails_without_it" end + create_table "whois_bodies", force: :cascade do |t| + t.integer "domain_id" + t.string "name" + t.text "whois_body" + t.json "whois_json" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "whois_bodies", ["domain_id"], name: "index_whois_bodies_on_domain_id", using: :btree + create_table "zonefile_settings", force: :cascade do |t| t.string "origin" t.integer "ttl" diff --git a/lib/tasks/whois.rake b/lib/tasks/whois.rake index 359db50da..a0489e336 100644 --- a/lib/tasks/whois.rake +++ b/lib/tasks/whois.rake @@ -14,9 +14,9 @@ namespace :whois do task export: :environment do start = Time.zone.now.to_f print "-----> Delete whois database data and sync with Registry master database..." - domains = Domain.pluck(:name, :whois_body, :whois_json) + whois_bodies = WhoisBody.pluck(:name, :whois_body, :whois_json) Whois::Domain.delete_all - Whois::Domain.import([:name, :whois_body, :whois_json], domains) + Whois::Domain.import([:name, :whois_body, :whois_json], whois_bodies) puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index 90f1a156e..12d5a3752 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -34,13 +34,9 @@ describe Domain do @domain.versions.should == [] end - it 'should not have whois_body' do + it 'should not have whois body' do @domain.whois_body.should == nil end - - it 'should not have whois json' do - @domain.whois_json.should == nil - end end context 'with valid attributes' do @@ -82,7 +78,7 @@ describe Domain do end it 'should have whois json by default' do - @domain.whois_json.present?.should == true + @domain.whois_body.whois_json.present?.should == true end it 'should have whois_body present by default' do @@ -113,7 +109,7 @@ describe Domain do @domain.nameservers = [ns1, ns2] @domain.update_whois_body - @domain.whois_body.should == <<-EOS + @domain.whois_body.whois_body.should == <<-EOS Estonia .ee Top Level Domain WHOIS server Domain: From a0c880151b0ef5dd8069a184758d4652ea3e2540 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Wed, 22 Apr 2015 17:13:10 +0300 Subject: [PATCH 06/11] Refactored whois_body to whois_record --- Guardfile | 4 +-- app/models/domain.rb | 16 +++++------ app/models/whois/domain.rb | 5 ---- app/models/whois/record.rb | 5 ++++ app/models/{whois_body.rb => whois_record.rb} | 16 +++++------ .../20150422134243_rename_whois_body.rb | 9 +++++++ db/schema.rb | 10 +++---- db/whois_schema.rb | 10 +++---- lib/tasks/whois.rake | 27 ++++++++++++++----- spec/models/domain_spec.rb | 12 ++++----- 10 files changed, 68 insertions(+), 46 deletions(-) delete mode 100644 app/models/whois/domain.rb create mode 100644 app/models/whois/record.rb rename app/models/{whois_body.rb => whois_record.rb} (91%) create mode 100644 db/migrate/20150422134243_rename_whois_body.rb diff --git a/Guardfile b/Guardfile index ab7619ed7..31958aa77 100644 --- a/Guardfile +++ b/Guardfile @@ -9,8 +9,8 @@ group :red_green_refactor, halt_on_fail: true do # watch(%r{^(config|lib)/.*}) # end - # guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do - guard :rspec, cmd: 'spring rspec', notification: false do + guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do + # guard :rspec, cmd: 'spring rspec', notification: false do watch(%r{^spec/.+_spec\.rb$}) watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } watch('spec/spec_helper.rb') { "spec" } diff --git a/app/models/domain.rb b/app/models/domain.rb index ae8e24540..a094f48ae 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -33,7 +33,7 @@ class Domain < ActiveRecord::Base has_many :dnskeys, dependent: :destroy has_many :keyrelays - has_one :whois_body, dependent: :destroy + has_one :whois_record, dependent: :destroy accepts_nested_attributes_for :dnskeys, allow_destroy: true @@ -53,7 +53,7 @@ class Domain < ActiveRecord::Base self.updated_at = Time.zone.now end after_save :manage_automatic_statuses - after_save :update_whois_body + after_save :update_whois_record after_save :update_whois_server validates :name_dirty, domain_name: true, uniqueness: true @@ -125,7 +125,7 @@ class Domain < ActiveRecord::Base includes( :registrar, :nameservers, - :whois_body, + :whois_record, { tech_contacts: :registrar }, { admin_contacts: :registrar } ) @@ -243,14 +243,14 @@ class Domain < ActiveRecord::Base log end - def update_whois_body - self.whois_body = WhoisBody.create if whois_body.blank? - whois_body.update + def update_whois_record + self.whois_record = WhoisRecord.create if whois_record.blank? + whois_record.update end def update_whois_server - if whois_body.present? - whois_body.update_whois_server + if whois_record.present? + whois_record.update_whois_server else logger.info "NO WHOIS BODY for domain: #{name}" end diff --git a/app/models/whois/domain.rb b/app/models/whois/domain.rb deleted file mode 100644 index 5aa890af1..000000000 --- a/app/models/whois/domain.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Whois - class Domain < Whois::Server - self.table_name = 'domains' - end -end diff --git a/app/models/whois/record.rb b/app/models/whois/record.rb new file mode 100644 index 000000000..08a92b6e1 --- /dev/null +++ b/app/models/whois/record.rb @@ -0,0 +1,5 @@ +module Whois + class Record < Whois::Server + self.table_name = 'whois_records' + end +end diff --git a/app/models/whois_body.rb b/app/models/whois_record.rb similarity index 91% rename from app/models/whois_body.rb rename to app/models/whois_record.rb index 29e1917b9..d64cb58d2 100644 --- a/app/models/whois_body.rb +++ b/app/models/whois_record.rb @@ -1,11 +1,11 @@ -class WhoisBody < ActiveRecord::Base +class WhoisRecord < ActiveRecord::Base belongs_to :domain def update_whois_server - return logger.info "NO WHOIS NAME for whois_body id: #{id}" if name.blank? - wd = Whois::Domain.find_or_initialize_by(name: name) - wd.whois_body = whois_body - wd.whois_json = whois_json + return logger.info "NO WHOIS NAME for whois record id: #{id}" if name.blank? + wd = Whois::Record.find_or_initialize_by(name: name) + wd.body = body + wd.json = json wd.save end @@ -53,12 +53,12 @@ class WhoisBody < ActiveRecord::Base end self.name = h[:name] - self.whois_body = body - self.whois_json = h + self.body = generated_body + self.json = h save end - def body + def generated_body <<-EOS Estonia .ee Top Level Domain WHOIS server diff --git a/db/migrate/20150422134243_rename_whois_body.rb b/db/migrate/20150422134243_rename_whois_body.rb new file mode 100644 index 000000000..657b1a762 --- /dev/null +++ b/db/migrate/20150422134243_rename_whois_body.rb @@ -0,0 +1,9 @@ +class RenameWhoisBody < ActiveRecord::Migration + def change + rename_column :whois_bodies, :whois_body, :body + rename_column :whois_bodies, :whois_json, :json + remove_index :whois_bodies, :domain_id + rename_table :whois_bodies, :whois_records + add_index :whois_records, :domain_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 3ecb2783b..62520c706 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150423083308) do +ActiveRecord::Schema.define(version: 20150422134243) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -910,16 +910,16 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.text "depricated_table_but_somehow_paper_trail_tests_fails_without_it" end - create_table "whois_bodies", force: :cascade do |t| + create_table "whois_records", force: :cascade do |t| t.integer "domain_id" t.string "name" - t.text "whois_body" - t.json "whois_json" + t.text "body" + t.json "json" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - add_index "whois_bodies", ["domain_id"], name: "index_whois_bodies_on_domain_id", using: :btree + add_index "whois_records", ["domain_id"], name: "index_whois_records_on_domain_id", using: :btree create_table "zonefile_settings", force: :cascade do |t| t.string "origin" diff --git a/db/whois_schema.rb b/db/whois_schema.rb index 0ddeb58b3..ffd951e79 100644 --- a/db/whois_schema.rb +++ b/db/whois_schema.rb @@ -14,16 +14,16 @@ ActiveRecord::Schema.define(version: 20150113113236) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" + # enable_extension "plpgsql" - create_table "domains", force: :cascade do |t| + create_table "whois_records", force: :cascade do |t| t.string "name" - t.text "whois_body" - t.json "whois_json" + t.text "body" + t.json "json" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - add_index "domains", ["name"], name: "index_domains_on_name", using: :btree + add_index "whois_records", ["name"], name: "index_domains_on_name", using: :btree end diff --git a/lib/tasks/whois.rake b/lib/tasks/whois.rake index a0489e336..9a62dee9a 100644 --- a/lib/tasks/whois.rake +++ b/lib/tasks/whois.rake @@ -1,10 +1,10 @@ namespace :whois do - desc 'Regenerate whois_body and whois_json at Registry master database (slow)' + desc 'Regenerate whois records at Registry master database (slow)' task regenerate: :environment do start = Time.zone.now.to_f - print "-----> Regenerate whois_body and whois_json at Registry master database..." + print "-----> Regenerate whois records at Registry master database..." Domain.included.find_each(batch_size: 50000).with_index do |d, index| - d.update_whois_body + d.update_whois_record print '.' if index % 100 == 0 end puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" @@ -14,14 +14,14 @@ namespace :whois do task export: :environment do start = Time.zone.now.to_f print "-----> Delete whois database data and sync with Registry master database..." - whois_bodies = WhoisBody.pluck(:name, :whois_body, :whois_json) - Whois::Domain.delete_all - Whois::Domain.import([:name, :whois_body, :whois_json], whois_bodies) + whois_records = WhoisRecord.pluck(:name, :body, :json) + Whois::Record.delete_all + Whois::Record.import([:name, :body, :json], whois_records) puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end namespace :schema do - desc 'Load whois schema' + desc 'Load whois schema into empty whois database' task load: [:environment] do whois_db = "whois_#{Rails.env}" begin @@ -37,5 +37,18 @@ namespace :whois do puts "\n#{e}" end end + + desc 'Force whois schema into exsisting whois database' + task force_load: [:environment] do + whois_db = "whois_#{Rails.env}" + begin + puts "\n------------------------ #{whois_db} schema loading ------------------------------\n" + ActiveRecord::Base.clear_all_connections! + ActiveRecord::Base.establish_connection(whois_db.to_sym) + load("#{Rails.root}/db/#{schema_file(whois_db)}") + rescue => e + puts "\n#{e}" + end + end end end diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index 12d5a3752..d84881f14 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -35,7 +35,7 @@ describe Domain do end it 'should not have whois body' do - @domain.whois_body.should == nil + @domain.whois_record.should == nil end end @@ -74,14 +74,14 @@ describe Domain do end it 'should have whois body by default' do - @domain.whois_body.present?.should == true + @domain.whois_record.present?.should == true end it 'should have whois json by default' do - @domain.whois_body.whois_json.present?.should == true + @domain.whois_record.json.present?.should == true end - it 'should have whois_body present by default' do + it 'should have whois record present by default' do @domain.name = 'yeah.ee' @domain.updated_at = Time.zone.parse('2020.02.02 02:00') @domain.registered_at = Time.zone.parse('2000.01.01 9:00') @@ -108,8 +108,8 @@ describe Domain do ns2.updated_at = Time.zone.parse('1970.01.01') @domain.nameservers = [ns1, ns2] - @domain.update_whois_body - @domain.whois_body.whois_body.should == <<-EOS + @domain.update_whois_record + @domain.whois_record.body.should == <<-EOS Estonia .ee Top Level Domain WHOIS server Domain: From 60f4198094c399ba067c69129d1f5dd9acdefe5d Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 23 Apr 2015 12:52:27 +0300 Subject: [PATCH 07/11] Added doc typo to admin layout --- app/views/layouts/admin/application.haml | 1 + app/views/layouts/registrar/application.haml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/layouts/admin/application.haml b/app/views/layouts/admin/application.haml index b6d679bf1..016ebd3b7 100644 --- a/app/views/layouts/admin/application.haml +++ b/app/views/layouts/admin/application.haml @@ -1,3 +1,4 @@ +!!! 5 %html{lang: I18n.locale.to_s} %head %meta{charset: "utf-8"}/ diff --git a/app/views/layouts/registrar/application.haml b/app/views/layouts/registrar/application.haml index a5798e85e..51e3961e4 100644 --- a/app/views/layouts/registrar/application.haml +++ b/app/views/layouts/registrar/application.haml @@ -1,4 +1,4 @@ -!!! +!!! 5 %html{lang: I18n.locale.to_s} %head %meta{charset: "utf-8"}/ From dfe93bd3475713423eced4e882a7e75847b2dadc Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 24 Apr 2015 16:40:59 +0300 Subject: [PATCH 08/11] Update whois body --- app/models/whois_record.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/models/whois_record.rb b/app/models/whois_record.rb index d64cb58d2..928896236 100644 --- a/app/models/whois_record.rb +++ b/app/models/whois_record.rb @@ -15,6 +15,7 @@ class WhoisRecord < ActiveRecord::Base end def update + @disclosed = [] h[:name] = domain.name h[:registrant] = domain.registrant.name h[:status] = domain.domain_statuses.map(&:human_value).join(', ') @@ -28,6 +29,7 @@ class WhoisRecord < ActiveRecord::Base h[:registrar_update_at] = domain.registrar.updated_at.to_s(:db) h[:admin_contacts] = [] domain.admin_contacts.each do |ac| + @disclosed << [:email, ac.email] h[:admin_contacts] << { name: ac.name, email: ac.email, @@ -37,6 +39,7 @@ class WhoisRecord < ActiveRecord::Base end h[:tech_contacts] = [] domain.tech_contacts.each do |tc| + @disclosed << [:email, tc.email] h[:tech_contacts] << { name: tc.name, email: tc.email, @@ -52,6 +55,8 @@ class WhoisRecord < ActiveRecord::Base } end + h[:disclosed] = @disclosed + self.name = h[:name] self.body = generated_body self.json = h @@ -89,7 +94,7 @@ More information at http://internet.ee out << (admins.size > 1 ? "\nAdministrative contacts" : "\nAdministrative contact") admins.each do |c| out << "\n name: #{c[:name]}" - out << "\n e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n email: Not Disclosed - Visit www.internet.ee for webbased WHOIS" out << "\n registrar: #{c[:registrar]}" out << "\n created: #{c[:created_at]}" out << "\n" @@ -98,7 +103,7 @@ More information at http://internet.ee out << (techs.size > 1 ? "\nTechnical contacts" : "\nTechnical contact:") techs.each do |c| out << "\n name: #{c[:name]}" - out << "\n e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n email: Not Disclosed - Visit www.internet.ee for webbased WHOIS" out << "\n registrar: #{c[:registrar]}" out << "\n created: #{c[:created_at]}" out << "\n" From 370227677bba6cb4f174957293b209b6ea129bf2 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 24 Apr 2015 16:47:57 +0300 Subject: [PATCH 09/11] Added legal document config as required, otherwise migration not working --- config/initializers/env_required.rb | 1 + db/schema.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/initializers/env_required.rb b/config/initializers/env_required.rb index c79520166..f4275bec6 100644 --- a/config/initializers/env_required.rb +++ b/config/initializers/env_required.rb @@ -8,6 +8,7 @@ required = %w( ca_key_path ca_key_password webclient_ip + legal_documents_dir ) Figaro.require_keys(required) diff --git a/db/schema.rb b/db/schema.rb index 62520c706..7cbf92109 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150422134243) do +ActiveRecord::Schema.define(version: 20150423083308) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" From 8c78a8f37747f4ca25b3ec54b73e6c5f2efc9beb Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 24 Apr 2015 16:50:02 +0300 Subject: [PATCH 10/11] added bank statement dir to requirements --- config/initializers/env_required.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/initializers/env_required.rb b/config/initializers/env_required.rb index f4275bec6..96f3bb47a 100644 --- a/config/initializers/env_required.rb +++ b/config/initializers/env_required.rb @@ -9,6 +9,7 @@ required = %w( ca_key_password webclient_ip legal_documents_dir + bank_statement_import_dir ) Figaro.require_keys(required) From 33dfbfb7394153d22eb0e7d48026f18358ed5d38 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Fri, 24 Apr 2015 16:53:28 +0300 Subject: [PATCH 11/11] Update whois spec --- spec/models/domain_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index d84881f14..ac8dc8c10 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -124,13 +124,13 @@ Domain: Administrative contact name: First Admin - e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS + email: Not Disclosed - Visit www.internet.ee for webbased WHOIS registrar: First Registrar Ltd created: 2016-01-01 00:00:00 Technical contact: name: First Tech - e-mail: Not Disclosed - Visit www.internet.ee for webbased WHOIS + email: Not Disclosed - Visit www.internet.ee for webbased WHOIS registrar: First Registrar Ltd created: 2016-01-01 00:00:00