Merge branch 'master' of github.com:domify/registry

CHANGELOG.md

@@ -1,15 +1,19 @@
 24.04.2015
 
-  * Update zonefile procedure
+* Update zonefile procedure
 
 23.04.2015
-  
-  * Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup`
+
+* Add `bank_statement_import_dir: 'import/legal_documents'` to application.yml, run `mina setup`
 
 22.04.2015
 
 * Configure smtp (see application-example.yml)
 
+22.04.2015
+
+* Whois database schema updated. Please reset whois database and run `rake whois:schema:load`
+
 21.04.2015
 
 * Install packages for wkhtmltopdf (see readme)

Guardfile

@@ -9,8 +9,8 @@ group :red_green_refactor, halt_on_fail: true do
     # watch(%r{^(config|lib)/.*})
   # end
 
-  # guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do
-  guard :rspec, cmd: 'spring rspec', notification: false do
+  guard :rspec, cmd: 'spring rspec --fail-fast', notification: false do
+  # guard :rspec, cmd: 'spring rspec', notification: false do
     watch(%r{^spec/.+_spec\.rb$})
     watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
     watch('spec/spec_helper.rb')  { "spec" }

app/models/domain.rb

@@ -33,6 +33,7 @@ class Domain < ActiveRecord::Base
   has_many :dnskeys, dependent: :destroy
 
   has_many :keyrelays
+  has_one :whois_record, dependent: :destroy
 
   accepts_nested_attributes_for :dnskeys, allow_destroy: true
 
@@ -52,7 +53,7 @@ class Domain < ActiveRecord::Base
     self.updated_at = Time.zone.now
   end
   after_save :manage_automatic_statuses
-  after_save :update_whois_body
+  after_save :update_whois_record
   after_save :update_whois_server
 
   validates :name_dirty, domain_name: true, uniqueness: true
@@ -124,6 +125,7 @@ class Domain < ActiveRecord::Base
       includes(
         :registrar, 
         :nameservers, 
+        :whois_record,
         { tech_contacts: :registrar },
         { admin_contacts: :registrar }
       )
@@ -226,6 +228,9 @@ class Domain < ActiveRecord::Base
       domain_statuses.find_by(value: DomainStatus::OK).try(:destroy)
     end
 
+    # otherwise domain_statuses are in old state for domain object
+    domain_statuses.reload 
+
     # contacts.includes(:address).each(&:manage_statuses)
   end
 
@@ -234,64 +239,20 @@ class Domain < ActiveRecord::Base
     log[:admin_contacts] = admin_contacts.map(&:attributes)
     log[:tech_contacts]  = tech_contacts.map(&:attributes)
     log[:nameservers]    = nameservers.map(&:attributes)
-    log[:registrant]  = [registrant.try(:attributes)]
+    log[:registrant]     = [registrant.try(:attributes)]
     log
   end
 
-  # rubocop:disable Metrics/MethodLength
-  def update_whois_body
-    self.whois_body = <<-EOS
-Estonia .ee Top Level Domain WHOIS server
-
-  domain:     #{name}
-  registrant: #{registrant.name}
-  status:     #{domain_statuses.map(&:value).join(', ')}
-  registered: #{registered_at and registered_at.to_s(:db)}
-  changed:    #{updated_at and updated_at.to_s(:db)}
-  expire:     #{valid_to and valid_to.to_s(:db)}
-  outzone:
-  delete:
-
-  #{contacts_body}
-
-  nsset:
-  nserver:
-
-  registrar: #{registrar}
-  phone: #{registrar.phone}
-  address: #{registrar.address}
-  created: #{registrar.created_at.to_s(:db)}
-  changed: #{registrar.updated_at.to_s(:db)}
-
-Estonia .ee Top Level Domain WHOIS server
-More information at http://internet.ee
-    EOS
-  end
-  # rubocop:enable Metrics/MethodLength
-
-  def contacts_body
-    out = ''
-    admin_contacts.each do |c|
-      out << 'Admin contact:'
-      out << "name: #{c.name}"
-      out << "email: #{c.email}"
-      out << "registrar: #{c.registrar}"
-      out << "created: #{c.created_at.to_s(:db)}"
-    end
-
-    tech_contacts.each do |c|
-      out << 'Tech contact:'
-      out << "name: #{c.name}"
-      out << "email: #{c.email}"
-      out << "registrar: #{c.registrar}"
-      out << "created: #{c.created_at.to_s(:db)}"
-    end
-    out
+  def update_whois_record
+    self.whois_record = WhoisRecord.create if whois_record.blank?
+    whois_record.update
   end
 
   def update_whois_server
-    wd = Whois::Domain.find_or_initialize_by(name: name)
-    wd.whois_body = whois_body
-    wd.save
+    if whois_record.present?
+      whois_record.update_whois_server
+    else
+      logger.info "NO WHOIS BODY for domain: #{name}"
+    end
   end
 end

app/models/domain_status.rb

@@ -109,6 +109,15 @@ class DomainStatus < ActiveRecord::Base
     CLIENT_STATUSES.include?(value)
   end
 
+  def human_value
+    case value
+    when 'ok'
+      'ok (paid and in zone)'
+    else 
+      value
+    end
+  end
+
   class << self
     def statuses_for_client
       CLIENT_STATUSES.map { |x| x.sub('client', '') }

app/models/whois.rb

@@ -1,2 +0,0 @@
-module Whois
-end

app/models/whois/domain.rb

@@ -1,5 +0,0 @@
-module Whois
-  class Domain < Whois::Server
-    self.table_name = 'domains'
-  end
-end

app/models/whois/record.rb

@@ -0,0 +1,5 @@
+module Whois
+  class Record < Whois::Server
+    self.table_name = 'whois_records'
+  end
+end

app/models/whois_record.rb

@@ -0,0 +1,123 @@
+class WhoisRecord < ActiveRecord::Base
+  belongs_to :domain
+
+  def update_whois_server
+    return logger.info "NO WHOIS NAME for whois record id: #{id}" if name.blank?
+    wd = Whois::Record.find_or_initialize_by(name: name)
+    wd.body = body
+    wd.json = json
+    wd.save
+  end
+
+  # rubocop:disable Metrics/MethodLength
+  def h
+    @h ||= HashWithIndifferentAccess.new
+  end
+
+  def update
+    @disclosed = []
+    h[:name] = domain.name
+    h[:registrant] = domain.registrant.name
+    h[:status] = domain.domain_statuses.map(&:human_value).join(', ')
+    h[:registered] = domain.registered_at and domain.registered_at.to_s(:db)
+    h[:updated_at] = domain.updated_at and domain.updated_at.to_s(:db)
+    h[:valid_to] = domain.valid_to and domain.valid_to.to_s(:db)
+    
+    h[:registrar] = domain.registrar.name
+    h[:registrar_phone] = domain.registrar.phone
+    h[:registrar_address] = domain.registrar.address
+    h[:registrar_update_at] = domain.registrar.updated_at.to_s(:db)
+    h[:admin_contacts] = []
+    domain.admin_contacts.each do |ac|
+      @disclosed << [:email, ac.email]
+      h[:admin_contacts] << {
+        name: ac.name,
+        email: ac.email,
+        registrar: ac.registrar.name,
+        created_at: ac.created_at.to_s(:db)
+      }
+    end
+    h[:tech_contacts] = []
+    domain.tech_contacts.each do |tc|
+      @disclosed << [:email, tc.email]
+      h[:tech_contacts] << {
+        name: tc.name,
+        email: tc.email,
+        registrar: tc.registrar.name,
+        created_at: tc.created_at.to_s(:db)
+      }
+    end
+    h[:nameservers] = []
+    domain.nameservers.each do |ns|
+      h[:nameservers] << {
+        hostname: ns.hostname,
+        updated_at: ns.updated_at.to_s(:db)
+      }
+    end
+
+    h[:disclosed] = @disclosed
+
+    self.name = h[:name]
+    self.body = generated_body
+    self.json = h
+    save
+  end
+
+  def generated_body
+    <<-EOS
+Estonia .ee Top Level Domain WHOIS server
+
+Domain:
+  name:       #{h[:name]}
+  registrant: #{h[:registrant]}
+  status:     #{h[:status]}
+  registered: #{h[:registered]}
+  changed:    #{h[:updated_at]}
+  expire:     #{h[:valid_to]}
+  outzone:
+  delete:
+#{contacts_body(h[:admin_contacts], h[:tech_contacts])}
+Registrar:
+  name:       #{h[:registrar]}
+  phone:      #{h[:registrar_phone]}
+  address:    #{h[:registrar_address]}
+  changed:    #{h[:registrar_update_at]}
+#{nameservers_body(h[:nameservers])}
+Estonia .ee Top Level Domain WHOIS server
+More information at http://internet.ee
+    EOS
+  end
+  # rubocop:enable Metrics/MethodLength
+
+  def contacts_body(admins, techs)
+    out = ''
+    out << (admins.size > 1 ? "\nAdministrative contacts" : "\nAdministrative contact")
+    admins.each do |c|
+      out << "\n  name:       #{c[:name]}"
+      out << "\n  email:      Not Disclosed - Visit www.internet.ee for webbased WHOIS"
+      out << "\n  registrar:  #{c[:registrar]}"
+      out << "\n  created:    #{c[:created_at]}"
+      out << "\n"
+    end
+
+    out << (techs.size > 1 ? "\nTechnical contacts" : "\nTechnical contact:")
+    techs.each do |c|
+      out << "\n  name:       #{c[:name]}"
+      out << "\n  email:      Not Disclosed - Visit www.internet.ee for webbased WHOIS"
+      out << "\n  registrar:  #{c[:registrar]}"
+      out << "\n  created:    #{c[:created_at]}"
+      out << "\n"
+    end
+    out
+  end
+
+  def nameservers_body(nservers)
+    out = "\nName servers:"
+    nservers.each do |ns|
+      out << "\n  nserver:   #{ns[:hostname]}"
+      out << "\n  changed:   #{ns[:updated_at]}"
+      out << "\n"
+    end
+    out
+  end
+end

app/views/layouts/admin/application.haml

@@ -1,3 +1,4 @@
+!!! 5
 %html{lang: I18n.locale.to_s}
   %head
     %meta{charset: "utf-8"}/

app/views/layouts/registrar/application.haml

@@ -1,4 +1,4 @@
-!!!
+!!! 5
 %html{lang: I18n.locale.to_s}
   %head
     %meta{charset: "utf-8"}/

config/initializers/env_required.rb

@@ -8,6 +8,8 @@ required = %w(
   ca_key_path
   ca_key_password
   webclient_ip
+  legal_documents_dir
+  bank_statement_import_dir
 )
 
 Figaro.require_keys(required)

db/migrate/20150421134820_add_whosi_json_body.rb

@@ -0,0 +1,5 @@
+class AddWhosiJsonBody < ActiveRecord::Migration
+  def change
+    add_column :domains, :whois_json, :json
+  end
+end

db/migrate/20150422092514_add_whois_body_to_registry.rb

@@ -0,0 +1,15 @@
+class AddWhoisBodyToRegistry < ActiveRecord::Migration
+  def change
+    create_table :whois_bodies, force: :cascade do |t|
+      t.integer  :domain_id
+      t.string   :name
+      t.text     :whois_body
+      t.json     :whois_json
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+    end
+    add_index  :whois_bodies, :domain_id
+    remove_column :domains, :whois_body, :text
+    remove_column :domains, :whois_json, :json
+  end
+end

db/migrate/20150422134243_rename_whois_body.rb

@@ -0,0 +1,9 @@
+class RenameWhoisBody < ActiveRecord::Migration
+  def change
+    rename_column :whois_bodies, :whois_body, :body
+    rename_column :whois_bodies, :whois_json, :json
+    remove_index :whois_bodies, :domain_id
+    rename_table :whois_bodies, :whois_records
+    add_index :whois_records, :domain_id
+  end
+end

db/schema.rb

@@ -294,12 +294,15 @@ ActiveRecord::Schema.define(version: 20150423083308) do
     t.string   "period_unit",          limit: 1
     t.string   "creator_str"
     t.string   "updator_str"
-    t.text     "whois_body"
     t.integer  "legacy_id"
     t.integer  "legacy_registrar_id"
     t.integer  "legacy_registrant_id"
+    t.datetime "outzone_at"
+    t.datetime "delete_at"
   end
 
+  add_index "domains", ["delete_at"], name: "index_domains_on_delete_at", using: :btree
+  add_index "domains", ["outzone_at"], name: "index_domains_on_outzone_at", using: :btree
   add_index "domains", ["registrant_id"], name: "index_domains_on_registrant_id", using: :btree
   add_index "domains", ["registrar_id"], name: "index_domains_on_registrar_id", using: :btree
 
@@ -907,6 +910,17 @@ ActiveRecord::Schema.define(version: 20150423083308) do
     t.text "depricated_table_but_somehow_paper_trail_tests_fails_without_it"
   end
 
+  create_table "whois_records", force: :cascade do |t|
+    t.integer  "domain_id"
+    t.string   "name"
+    t.text     "body"
+    t.json     "json"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "whois_records", ["domain_id"], name: "index_whois_records_on_domain_id", using: :btree
+
   create_table "zonefile_settings", force: :cascade do |t|
     t.string   "origin"
     t.integer  "ttl"

db/seeds.rb

@@ -39,29 +39,35 @@ ApiUser.where(
   registrar: registrar2
 ).first_or_create!
 
-AdminUser.where(
+admin1 = {
   username: 'user1',
   password: 'test1',
   email: 'user1@example.ee',
   identity_code: '37810013855',
   country_code: 'EE'
-).first_or_create!
-
-AdminUser.where(
+}
+admin2 = {
   username: 'user2',
   password: 'test2',
   email: 'user2@example.ee',
   identity_code: '37810010085',
   country_code: 'EE'
-).first_or_create!
-
-AdminUser.where(
+}
+admin3 = {
   username: 'user3',
   password: 'test3',
   email: 'user3@example.ee',
   identity_code: '37810010727',
   country_code: 'EE'
-).first_or_create!
+}
+
+[admin1, admin2, admin3].each do |at|
+  admin = AdminUser.where(at)
+  next if admin.present?
+  admin = AdminUser.new(at)
+  admin.roles = ['admin']
+  admin.save
+end
 
 ZonefileSetting.where({
   origin: 'ee',
@@ -85,8 +91,6 @@ ZonefileSetting.where({
   master_nameserver: 'ns.tld.ee'
 }).first_or_create!
 
-AdminUser.update_all(roles: ['admin'])
-
 Registrar.where(
   name: 'EIS',
   reg_no: '90010019',

db/whois_schema.rb

@@ -14,15 +14,16 @@
 ActiveRecord::Schema.define(version: 20150113113236) do
 
   # These are extensions that must be enabled in order to support this database
-  enable_extension "plpgsql"
+  # enable_extension "plpgsql"
 
-  create_table "domains", force: :cascade do |t|
+  create_table "whois_records", force: :cascade do |t|
     t.string   "name"
-    t.text     "whois_body"
+    t.text     "body"
+    t.json     "json"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
   end
 
-  add_index "domains", ["name"], name: "index_domains_on_name", using: :btree
+  add_index "whois_records", ["name"], name: "index_domains_on_name", using: :btree
 
 end

lib/tasks/db.rake

@@ -33,7 +33,7 @@ namespace :db do
     task create: [:environment, :load_config] do
       databases.each do |name|
         begin
-          puts "\n---------------------------- Create #{name} ----------------------------------------\n"
+          puts "\n------------------------ Create #{name} ---------------------------------------\n"
           ActiveRecord::Base.clear_all_connections!
           conf = ActiveRecord::Base.configurations
           
@@ -75,7 +75,7 @@ namespace :db do
       task load: [:environment, :load_config] do
         databases.each do |name|
           begin
-            puts "\n---------------------------- #{name} schema loaded ----------------------------------------\n"
+            puts "\n------------------------ #{name} schema loading -----------------------------\n"
             ActiveRecord::Base.clear_all_connections!
             ActiveRecord::Base.establish_connection(name.to_sym)
             if ActiveRecord::Base.connection.table_exists?('schema_migrations')

lib/tasks/whois.rake

@@ -1,33 +1,54 @@
 namespace :whois do
-  desc 'Delete whois database data and import all from Registry (fast)'
-  task reset: :environment do
+  desc 'Regenerate whois records at Registry master database (slow)'
+  task regenerate: :environment do
     start = Time.zone.now.to_f
-    print "-----> Reset whois database and sync..."
-    domains = Domain.pluck(:name, :whois_body)
-    Whois::Domain.delete_all
-    Whois::Domain.import([:name, :whois_body], domains)
-    puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds"
-  end
-
-  desc 'Sync whois database without reset (slow)'
-  task sync: :environment do
-    start = Time.zone.now.to_f
-    print "-----> Sync whois database..."
-    Domain.select(:id, :name, :whois_body).find_each(batch_size: 100000).with_index do |d, index|
-      d.update_whois_server
+    print "-----> Regenerate whois records at Registry master database..."
+    Domain.included.find_each(batch_size: 50000).with_index do |d, index|
+      d.update_whois_record
       print '.' if index % 100 == 0
     end
     puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds"
   end
 
-  desc 'Regenerate whois_body at Registry master database (slow)'
-  task generate: :environment do
+  desc 'Delete whois database data and sync with Registry master database (fast)'
+  task export: :environment do
     start = Time.zone.now.to_f
-    print "-----> Update Registry records..."
-    Domain.included.find_each(batch_size: 100000).with_index do |d, index|
-      d.update_columns(whois_body: d.update_whois_body)
-      print '.' if index % 100 == 0
-    end
+    print "-----> Delete whois database data and sync with Registry master database..."
+    whois_records = WhoisRecord.pluck(:name, :body, :json)
+    Whois::Record.delete_all
+    Whois::Record.import([:name, :body, :json], whois_records)
     puts "\n-----> all done in #{(Time.zone.now.to_f - start).round(2)} seconds"
   end
+
+  namespace :schema do
+    desc 'Load whois schema into empty whois database'
+    task load: [:environment] do
+      whois_db = "whois_#{Rails.env}"
+      begin
+        puts "\n------------------------ #{whois_db} schema loading ------------------------------\n"
+        ActiveRecord::Base.clear_all_connections!
+        ActiveRecord::Base.establish_connection(whois_db.to_sym)
+        if ActiveRecord::Base.connection.table_exists?('schema_migrations')
+          puts 'Found tables, skip schema load!'
+        else
+          load("#{Rails.root}/db/#{schema_file(whois_db)}")
+        end
+      rescue => e
+        puts "\n#{e}"
+      end
+    end
+
+    desc 'Force whois schema into exsisting whois database'
+    task force_load: [:environment] do
+      whois_db = "whois_#{Rails.env}"
+      begin
+        puts "\n------------------------ #{whois_db} schema loading ------------------------------\n"
+        ActiveRecord::Base.clear_all_connections!
+        ActiveRecord::Base.establish_connection(whois_db.to_sym)
+        load("#{Rails.root}/db/#{schema_file(whois_db)}")
+      rescue => e
+        puts "\n#{e}"
+      end
+    end
+  end
 end

spec/models/domain_spec.rb

@@ -34,10 +34,9 @@ describe Domain do
       @domain.versions.should == []
     end
 
-    it 'should not have whois_body' do
-      @domain.whois_body.should == nil
+    it 'should not have whois body' do
+      @domain.whois_record.should == nil
     end
-
   end
 
   context 'with valid attributes' do
@@ -74,8 +73,83 @@ describe Domain do
       domain.errors.full_messages.should match_array(["Admin domain contacts is invalid"])
     end
 
-    it 'should not have whois_body present by default' do
-      @domain.whois_body.present?.should == true
+    it 'should have whois body by default' do
+      @domain.whois_record.present?.should == true
+    end
+
+    it 'should have whois json by default' do
+      @domain.whois_record.json.present?.should == true
+    end
+
+    it 'should have whois record present by default' do
+      @domain.name = 'yeah.ee'
+      @domain.updated_at    = Time.zone.parse('2020.02.02 02:00')
+      @domain.registered_at = Time.zone.parse('2000.01.01 9:00')
+      @domain.valid_to      = Time.zone.parse('2016.04.21 0:00')
+      registrar = Fabricate(:registrar, 
+                            name: 'First Registrar Ltd', 
+                            created_at: Time.zone.parse('1995.01.01'),
+                            updated_at: Time.zone.parse('1996.01.01'))
+      @domain.registrant    = Fabricate(:contact, 
+                                        name: 'Jarren Jakubowski0', 
+                                        created_at: Time.zone.parse('2005.01.01'))
+      @domain.admin_contacts = [Fabricate(:contact,
+                                          name: 'First Admin', 
+                                          registrar: registrar,
+                                          created_at: Time.zone.parse('2016.01.01'))]
+      @domain.tech_contacts = [Fabricate(:contact,
+                                         name: 'First Tech', 
+                                         registrar: registrar,
+                                         created_at: Time.zone.parse('2016.01.01'))]
+      @domain.registrar = registrar
+      ns1 = Fabricate(:nameserver, hostname: 'test.ee')
+      ns1.updated_at = Time.zone.parse('1980.01.01')
+      ns2 = Fabricate(:nameserver, hostname: 'test1.ee')
+      ns2.updated_at = Time.zone.parse('1970.01.01')
+      @domain.nameservers = [ns1, ns2]
+
+      @domain.update_whois_record
+      @domain.whois_record.body.should == <<-EOS
+Estonia .ee Top Level Domain WHOIS server
+
+Domain:
+  name:       yeah.ee
+  registrant: Jarren Jakubowski0
+  status:     ok (paid and in zone)
+  registered: 2000-01-01 09:00:00 UTC
+  changed:    2020-02-02 02:00:00 UTC
+  expire:     2016-04-21 00:00:00 UTC
+  outzone:
+  delete:
+
+Administrative contact
+  name:       First Admin
+  email:      Not Disclosed - Visit www.internet.ee for webbased WHOIS
+  registrar:  First Registrar Ltd
+  created:    2016-01-01 00:00:00
+
+Technical contact:
+  name:       First Tech
+  email:      Not Disclosed - Visit www.internet.ee for webbased WHOIS
+  registrar:  First Registrar Ltd
+  created:    2016-01-01 00:00:00
+
+Registrar:
+  name:       First Registrar Ltd
+  phone:      
+  address:    Street 999, Town, County, Postal
+  changed:    1996-01-01 00:00:00
+
+Name servers:
+  nserver:   test.ee
+  changed:   1980-01-01 00:00:00
+
+  nserver:   test1.ee
+  changed:   1970-01-01 00:00:00
+
+Estonia .ee Top Level Domain WHOIS server
+More information at http://internet.ee
+ EOS
     end
 
     context 'with versioning' do