diff --git a/CHANGELOG.md b/CHANGELOG.md index add9ce179..798980aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,19 @@ 24.04.2015 - * Update zonefile procedure +* Update zonefile procedure 23.04.2015 - - * Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup` + +* Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup` 22.04.2015 * Configure smtp (see application-example.yml) +22.04.2015 + +* Whois database schema updated. Please reset whois database and run `rake whois:schema:load` + 21.04.2015 * Install packages for wkhtmltopdf (see readme) diff --git a/Guardfile b/Guardfile index ab7619ed7..31958aa77 100644 --- a/Guardfile +++ b/Guardfile @@ -9,8 +9,8 @@ group :red_green_refactor, halt_on_fail: true do # watch(%r{^(config|lib)/.*}) # end - # guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do - guard :rspec, cmd: 'spring rspec', notification: false do + guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do + # guard :rspec, cmd: 'spring rspec', notification: false do watch(%r{^spec/.+_spec\.rb$}) watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } watch('spec/spec_helper.rb') { "spec" } diff --git a/app/models/domain.rb b/app/models/domain.rb index 3b71185eb..a094f48ae 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -33,6 +33,7 @@ class Domain < ActiveRecord::Base has_many :dnskeys, dependent: :destroy has_many :keyrelays + has_one :whois_record, dependent: :destroy accepts_nested_attributes_for :dnskeys, allow_destroy: true @@ -52,7 +53,7 @@ class Domain < ActiveRecord::Base self.updated_at = Time.zone.now end after_save :manage_automatic_statuses - after_save :update_whois_body + after_save :update_whois_record after_save :update_whois_server validates :name_dirty, domain_name: true, uniqueness: true @@ -124,6 +125,7 @@ class Domain < ActiveRecord::Base includes( :registrar, :nameservers, + :whois_record, { tech_contacts: :registrar }, { admin_contacts: :registrar } ) @@ -226,6 +228,9 @@ class Domain < ActiveRecord::Base domain_statuses.find_by(value: DomainStatus::OK).try(:destroy) end + # otherwise domain_statuses are in old state for domain object + domain_statuses.reload + # contacts.includes(:address).each(&:manage_statuses) end @@ -234,64 +239,20 @@ class Domain < ActiveRecord::Base log[:admin_contacts] = admin_contacts.map(&:attributes) log[:tech_contacts] = tech_contacts.map(&:attributes) log[:nameservers] = nameservers.map(&:attributes) - log[:registrant] = [registrant.try(:attributes)] + log[:registrant] = [registrant.try(:attributes)] log end - # rubocop:disable Metrics/MethodLength - def update_whois_body - self.whois_body = <<-EOS -Estonia .ee Top Level Domain WHOIS server - - domain: #{name} - registrant: #{registrant.name} - status: #{domain_statuses.map(&:value).join(', ')} - registered: #{registered_at and registered_at.to_s(:db)} - changed: #{updated_at and updated_at.to_s(:db)} - expire: #{valid_to and valid_to.to_s(:db)} - outzone: - delete: - - #{contacts_body} - - nsset: - nserver: - - registrar: #{registrar} - phone: #{registrar.phone} - address: #{registrar.address} - created: #{registrar.created_at.to_s(:db)} - changed: #{registrar.updated_at.to_s(:db)} - -Estonia .ee Top Level Domain WHOIS server -More information at http://internet.ee - EOS - end - # rubocop:enable Metrics/MethodLength - - def contacts_body - out = '' - admin_contacts.each do |c| - out << 'Admin contact:' - out << "name: #{c.name}" - out << "email: #{c.email}" - out << "registrar: #{c.registrar}" - out << "created: #{c.created_at.to_s(:db)}" - end - - tech_contacts.each do |c| - out << 'Tech contact:' - out << "name: #{c.name}" - out << "email: #{c.email}" - out << "registrar: #{c.registrar}" - out << "created: #{c.created_at.to_s(:db)}" - end - out + def update_whois_record + self.whois_record = WhoisRecord.create if whois_record.blank? + whois_record.update end def update_whois_server - wd = Whois::Domain.find_or_initialize_by(name: name) - wd.whois_body = whois_body - wd.save + if whois_record.present? + whois_record.update_whois_server + else + logger.info "NO WHOIS BODY for domain: #{name}" + end end end diff --git a/app/models/domain_status.rb b/app/models/domain_status.rb index 83bbf1308..5a435ecc4 100644 --- a/app/models/domain_status.rb +++ b/app/models/domain_status.rb @@ -109,6 +109,15 @@ class DomainStatus < ActiveRecord::Base CLIENT_STATUSES.include?(value) end + def human_value + case value + when 'ok' + 'ok (paid and in zone)' + else + value + end + end + class << self def statuses_for_client CLIENT_STATUSES.map { |x| x.sub('client', '') } diff --git a/app/models/whois.rb b/app/models/whois.rb deleted file mode 100644 index 1364d5099..000000000 --- a/app/models/whois.rb +++ /dev/null @@ -1,2 +0,0 @@ -module Whois -end diff --git a/app/models/whois/domain.rb b/app/models/whois/domain.rb deleted file mode 100644 index 5aa890af1..000000000 --- a/app/models/whois/domain.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Whois - class Domain < Whois::Server - self.table_name = 'domains' - end -end diff --git a/app/models/whois/record.rb b/app/models/whois/record.rb new file mode 100644 index 000000000..08a92b6e1 --- /dev/null +++ b/app/models/whois/record.rb @@ -0,0 +1,5 @@ +module Whois + class Record < Whois::Server + self.table_name = 'whois_records' + end +end diff --git a/app/models/whois_record.rb b/app/models/whois_record.rb new file mode 100644 index 000000000..928896236 --- /dev/null +++ b/app/models/whois_record.rb @@ -0,0 +1,123 @@ +class WhoisRecord < ActiveRecord::Base + belongs_to :domain + + def update_whois_server + return logger.info "NO WHOIS NAME for whois record id: #{id}" if name.blank? + wd = Whois::Record.find_or_initialize_by(name: name) + wd.body = body + wd.json = json + wd.save + end + + # rubocop:disable Metrics/MethodLength + def h + @h ||= HashWithIndifferentAccess.new + end + + def update + @disclosed = [] + h[:name] = domain.name + h[:registrant] = domain.registrant.name + h[:status] = domain.domain_statuses.map(&:human_value).join(', ') + h[:registered] = domain.registered_at and domain.registered_at.to_s(:db) + h[:updated_at] = domain.updated_at and domain.updated_at.to_s(:db) + h[:valid_to] = domain.valid_to and domain.valid_to.to_s(:db) + + h[:registrar] = domain.registrar.name + h[:registrar_phone] = domain.registrar.phone + h[:registrar_address] = domain.registrar.address + h[:registrar_update_at] = domain.registrar.updated_at.to_s(:db) + h[:admin_contacts] = [] + domain.admin_contacts.each do |ac| + @disclosed << [:email, ac.email] + h[:admin_contacts] << { + name: ac.name, + email: ac.email, + registrar: ac.registrar.name, + created_at: ac.created_at.to_s(:db) + } + end + h[:tech_contacts] = [] + domain.tech_contacts.each do |tc| + @disclosed << [:email, tc.email] + h[:tech_contacts] << { + name: tc.name, + email: tc.email, + registrar: tc.registrar.name, + created_at: tc.created_at.to_s(:db) + } + end + h[:nameservers] = [] + domain.nameservers.each do |ns| + h[:nameservers] << { + hostname: ns.hostname, + updated_at: ns.updated_at.to_s(:db) + } + end + + h[:disclosed] = @disclosed + + self.name = h[:name] + self.body = generated_body + self.json = h + save + end + + def generated_body + <<-EOS +Estonia .ee Top Level Domain WHOIS server + +Domain: + name: #{h[:name]} + registrant: #{h[:registrant]} + status: #{h[:status]} + registered: #{h[:registered]} + changed: #{h[:updated_at]} + expire: #{h[:valid_to]} + outzone: + delete: +#{contacts_body(h[:admin_contacts], h[:tech_contacts])} +Registrar: + name: #{h[:registrar]} + phone: #{h[:registrar_phone]} + address: #{h[:registrar_address]} + changed: #{h[:registrar_update_at]} +#{nameservers_body(h[:nameservers])} +Estonia .ee Top Level Domain WHOIS server +More information at http://internet.ee + EOS + end + # rubocop:enable Metrics/MethodLength + + def contacts_body(admins, techs) + out = '' + out << (admins.size > 1 ? "\nAdministrative contacts" : "\nAdministrative contact") + admins.each do |c| + out << "\n name: #{c[:name]}" + out << "\n email: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n registrar: #{c[:registrar]}" + out << "\n created: #{c[:created_at]}" + out << "\n" + end + + out << (techs.size > 1 ? "\nTechnical contacts" : "\nTechnical contact:") + techs.each do |c| + out << "\n name: #{c[:name]}" + out << "\n email: Not Disclosed - Visit www.internet.ee for webbased WHOIS" + out << "\n registrar: #{c[:registrar]}" + out << "\n created: #{c[:created_at]}" + out << "\n" + end + out + end + + def nameservers_body(nservers) + out = "\nName servers:" + nservers.each do |ns| + out << "\n nserver: #{ns[:hostname]}" + out << "\n changed: #{ns[:updated_at]}" + out << "\n" + end + out + end +end diff --git a/app/views/layouts/admin/application.haml b/app/views/layouts/admin/application.haml index b6d679bf1..016ebd3b7 100644 --- a/app/views/layouts/admin/application.haml +++ b/app/views/layouts/admin/application.haml @@ -1,3 +1,4 @@ +!!! 5 %html{lang: I18n.locale.to_s} %head %meta{charset: "utf-8"}/ diff --git a/app/views/layouts/registrar/application.haml b/app/views/layouts/registrar/application.haml index a5798e85e..51e3961e4 100644 --- a/app/views/layouts/registrar/application.haml +++ b/app/views/layouts/registrar/application.haml @@ -1,4 +1,4 @@ -!!! +!!! 5 %html{lang: I18n.locale.to_s} %head %meta{charset: "utf-8"}/ diff --git a/config/initializers/env_required.rb b/config/initializers/env_required.rb index c79520166..96f3bb47a 100644 --- a/config/initializers/env_required.rb +++ b/config/initializers/env_required.rb @@ -8,6 +8,8 @@ required = %w( ca_key_path ca_key_password webclient_ip + legal_documents_dir + bank_statement_import_dir ) Figaro.require_keys(required) diff --git a/db/migrate/20150421134820_add_whosi_json_body.rb b/db/migrate/20150421134820_add_whosi_json_body.rb new file mode 100644 index 000000000..c1bf00075 --- /dev/null +++ b/db/migrate/20150421134820_add_whosi_json_body.rb @@ -0,0 +1,5 @@ +class AddWhosiJsonBody < ActiveRecord::Migration + def change + add_column :domains, :whois_json, :json + end +end diff --git a/db/migrate/20150422092514_add_whois_body_to_registry.rb b/db/migrate/20150422092514_add_whois_body_to_registry.rb new file mode 100644 index 000000000..6791bbcf9 --- /dev/null +++ b/db/migrate/20150422092514_add_whois_body_to_registry.rb @@ -0,0 +1,15 @@ +class AddWhoisBodyToRegistry < ActiveRecord::Migration + def change + create_table :whois_bodies, force: :cascade do |t| + t.integer :domain_id + t.string :name + t.text :whois_body + t.json :whois_json + t.datetime :created_at, null: false + t.datetime :updated_at, null: false + end + add_index :whois_bodies, :domain_id + remove_column :domains, :whois_body, :text + remove_column :domains, :whois_json, :json + end +end diff --git a/db/migrate/20150422134243_rename_whois_body.rb b/db/migrate/20150422134243_rename_whois_body.rb new file mode 100644 index 000000000..657b1a762 --- /dev/null +++ b/db/migrate/20150422134243_rename_whois_body.rb @@ -0,0 +1,9 @@ +class RenameWhoisBody < ActiveRecord::Migration + def change + rename_column :whois_bodies, :whois_body, :body + rename_column :whois_bodies, :whois_json, :json + remove_index :whois_bodies, :domain_id + rename_table :whois_bodies, :whois_records + add_index :whois_records, :domain_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 755d6eecc..7cbf92109 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -294,12 +294,15 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.string "period_unit", limit: 1 t.string "creator_str" t.string "updator_str" - t.text "whois_body" t.integer "legacy_id" t.integer "legacy_registrar_id" t.integer "legacy_registrant_id" + t.datetime "outzone_at" + t.datetime "delete_at" end + add_index "domains", ["delete_at"], name: "index_domains_on_delete_at", using: :btree + add_index "domains", ["outzone_at"], name: "index_domains_on_outzone_at", using: :btree add_index "domains", ["registrant_id"], name: "index_domains_on_registrant_id", using: :btree add_index "domains", ["registrar_id"], name: "index_domains_on_registrar_id", using: :btree @@ -907,6 +910,17 @@ ActiveRecord::Schema.define(version: 20150423083308) do t.text "depricated_table_but_somehow_paper_trail_tests_fails_without_it" end + create_table "whois_records", force: :cascade do |t| + t.integer "domain_id" + t.string "name" + t.text "body" + t.json "json" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "whois_records", ["domain_id"], name: "index_whois_records_on_domain_id", using: :btree + create_table "zonefile_settings", force: :cascade do |t| t.string "origin" t.integer "ttl" diff --git a/db/seeds.rb b/db/seeds.rb index 03dc6c18f..892cc9dfe 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -39,29 +39,35 @@ ApiUser.where( registrar: registrar2 ).first_or_create! -AdminUser.where( +admin1 = { username: 'user1', password: 'test1', email: 'user1@example.ee', identity_code: '37810013855', country_code: 'EE' -).first_or_create! - -AdminUser.where( +} +admin2 = { username: 'user2', password: 'test2', email: 'user2@example.ee', identity_code: '37810010085', country_code: 'EE' -).first_or_create! - -AdminUser.where( +} +admin3 = { username: 'user3', password: 'test3', email: 'user3@example.ee', identity_code: '37810010727', country_code: 'EE' -).first_or_create! +} + +[admin1, admin2, admin3].each do |at| + admin = AdminUser.where(at) + next if admin.present? + admin = AdminUser.new(at) + admin.roles = ['admin'] + admin.save +end ZonefileSetting.where({ origin: 'ee', @@ -85,8 +91,6 @@ ZonefileSetting.where({ master_nameserver: 'ns.tld.ee' }).first_or_create! -AdminUser.update_all(roles: ['admin']) - Registrar.where( name: 'EIS', reg_no: '90010019', diff --git a/db/whois_schema.rb b/db/whois_schema.rb index 0290bd178..ffd951e79 100644 --- a/db/whois_schema.rb +++ b/db/whois_schema.rb @@ -14,15 +14,16 @@ ActiveRecord::Schema.define(version: 20150113113236) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" + # enable_extension "plpgsql" - create_table "domains", force: :cascade do |t| + create_table "whois_records", force: :cascade do |t| t.string "name" - t.text "whois_body" + t.text "body" + t.json "json" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - add_index "domains", ["name"], name: "index_domains_on_name", using: :btree + add_index "whois_records", ["name"], name: "index_domains_on_name", using: :btree end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index aa06aa273..cd022bcb1 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -33,7 +33,7 @@ namespace :db do task create: [:environment, :load_config] do databases.each do |name| begin - puts "\n---------------------------- Create #{name} ----------------------------------------\n" + puts "\n------------------------ Create #{name} ---------------------------------------\n" ActiveRecord::Base.clear_all_connections! conf = ActiveRecord::Base.configurations @@ -75,7 +75,7 @@ namespace :db do task load: [:environment, :load_config] do databases.each do |name| begin - puts "\n---------------------------- #{name} schema loaded ----------------------------------------\n" + puts "\n------------------------ #{name} schema loading -----------------------------\n" ActiveRecord::Base.clear_all_connections! ActiveRecord::Base.establish_connection(name.to_sym) if ActiveRecord::Base.connection.table_exists?('schema_migrations') diff --git a/lib/tasks/whois.rake b/lib/tasks/whois.rake index 29d15fcb8..9a62dee9a 100644 --- a/lib/tasks/whois.rake +++ b/lib/tasks/whois.rake @@ -1,33 +1,54 @@ namespace :whois do - desc 'Delete whois database data and import all from Registry (fast)' - task reset: :environment do + desc 'Regenerate whois records at Registry master database (slow)' + task regenerate: :environment do start = Time.zone.now.to_f - print "-----> Reset whois database and sync..." - domains = Domain.pluck(:name, :whois_body) - Whois::Domain.delete_all - Whois::Domain.import([:name, :whois_body], domains) - puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" - end - - desc 'Sync whois database without reset (slow)' - task sync: :environment do - start = Time.zone.now.to_f - print "-----> Sync whois database..." - Domain.select(:id, :name, :whois_body).find_each(batch_size: 100000).with_index do |d, index| - d.update_whois_server + print "-----> Regenerate whois records at Registry master database..." + Domain.included.find_each(batch_size: 50000).with_index do |d, index| + d.update_whois_record print '.' if index % 100 == 0 end puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end - desc 'Regenerate whois_body at Registry master database (slow)' - task generate: :environment do + desc 'Delete whois database data and sync with Registry master database (fast)' + task export: :environment do start = Time.zone.now.to_f - print "-----> Update Registry records..." - Domain.included.find_each(batch_size: 100000).with_index do |d, index| - d.update_columns(whois_body: d.update_whois_body) - print '.' if index % 100 == 0 - end + print "-----> Delete whois database data and sync with Registry master database..." + whois_records = WhoisRecord.pluck(:name, :body, :json) + Whois::Record.delete_all + Whois::Record.import([:name, :body, :json], whois_records) puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds" end + + namespace :schema do + desc 'Load whois schema into empty whois database' + task load: [:environment] do + whois_db = "whois_#{Rails.env}" + begin + puts "\n------------------------ #{whois_db} schema loading ------------------------------\n" + ActiveRecord::Base.clear_all_connections! + ActiveRecord::Base.establish_connection(whois_db.to_sym) + if ActiveRecord::Base.connection.table_exists?('schema_migrations') + puts 'Found tables, skip schema load!' + else + load("#{Rails.root}/db/#{schema_file(whois_db)}") + end + rescue => e + puts "\n#{e}" + end + end + + desc 'Force whois schema into exsisting whois database' + task force_load: [:environment] do + whois_db = "whois_#{Rails.env}" + begin + puts "\n------------------------ #{whois_db} schema loading ------------------------------\n" + ActiveRecord::Base.clear_all_connections! + ActiveRecord::Base.establish_connection(whois_db.to_sym) + load("#{Rails.root}/db/#{schema_file(whois_db)}") + rescue => e + puts "\n#{e}" + end + end + end end diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index bb8881ff1..ac8dc8c10 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -34,10 +34,9 @@ describe Domain do @domain.versions.should == [] end - it 'should not have whois_body' do - @domain.whois_body.should == nil + it 'should not have whois body' do + @domain.whois_record.should == nil end - end context 'with valid attributes' do @@ -74,8 +73,83 @@ describe Domain do domain.errors.full_messages.should match_array(["Admin domain contacts is invalid"]) end - it 'should not have whois_body present by default' do - @domain.whois_body.present?.should == true + it 'should have whois body by default' do + @domain.whois_record.present?.should == true + end + + it 'should have whois json by default' do + @domain.whois_record.json.present?.should == true + end + + it 'should have whois record present by default' do + @domain.name = 'yeah.ee' + @domain.updated_at = Time.zone.parse('2020.02.02 02:00') + @domain.registered_at = Time.zone.parse('2000.01.01 9:00') + @domain.valid_to = Time.zone.parse('2016.04.21 0:00') + registrar = Fabricate(:registrar, + name: 'First Registrar Ltd', + created_at: Time.zone.parse('1995.01.01'), + updated_at: Time.zone.parse('1996.01.01')) + @domain.registrant = Fabricate(:contact, + name: 'Jarren Jakubowski0', + created_at: Time.zone.parse('2005.01.01')) + @domain.admin_contacts = [Fabricate(:contact, + name: 'First Admin', + registrar: registrar, + created_at: Time.zone.parse('2016.01.01'))] + @domain.tech_contacts = [Fabricate(:contact, + name: 'First Tech', + registrar: registrar, + created_at: Time.zone.parse('2016.01.01'))] + @domain.registrar = registrar + ns1 = Fabricate(:nameserver, hostname: 'test.ee') + ns1.updated_at = Time.zone.parse('1980.01.01') + ns2 = Fabricate(:nameserver, hostname: 'test1.ee') + ns2.updated_at = Time.zone.parse('1970.01.01') + @domain.nameservers = [ns1, ns2] + + @domain.update_whois_record + @domain.whois_record.body.should == <<-EOS +Estonia .ee Top Level Domain WHOIS server + +Domain: + name: yeah.ee + registrant: Jarren Jakubowski0 + status: ok (paid and in zone) + registered: 2000-01-01 09:00:00 UTC + changed: 2020-02-02 02:00:00 UTC + expire: 2016-04-21 00:00:00 UTC + outzone: + delete: + +Administrative contact + name: First Admin + email: Not Disclosed - Visit www.internet.ee for webbased WHOIS + registrar: First Registrar Ltd + created: 2016-01-01 00:00:00 + +Technical contact: + name: First Tech + email: Not Disclosed - Visit www.internet.ee for webbased WHOIS + registrar: First Registrar Ltd + created: 2016-01-01 00:00:00 + +Registrar: + name: First Registrar Ltd + phone: + address: Street 999, Town, County, Postal + changed: 1996-01-01 00:00:00 + +Name servers: + nserver: test.ee + changed: 1980-01-01 00:00:00 + + nserver: test1.ee + changed: 1970-01-01 00:00:00 + +Estonia .ee Top Level Domain WHOIS server +More information at http://internet.ee + EOS end context 'with versioning' do