Merge remote-tracking branch 'origin/104525318-history_import' into staging

Gemfile

@@ -12,6 +12,7 @@ gem 'rails',        '4.2.4' # when update, all initializers eis_custom files nee
 gem 'iso8601',      '0.8.6' # for dates and times
 gem 'hashie-forbidden_attributes', '0.1.1'
 gem 'SyslogLogger', '2.0', require: 'syslog/logger'
+gem 'parallel'
 
 # load env
 gem 'figaro', '1.1.1'

Gemfile.lock

@@ -330,6 +330,7 @@ GEM
     nprogress-rails (0.1.6.7)
     open4 (1.3.4)
     orm_adapter (0.5.0)
+    parallel (1.6.1)
     parser (2.2.2.6)
       ast (>= 1.1, < 3.0)
     pdfkit (0.6.2)
@@ -603,6 +604,7 @@ DEPENDENCIES
   nokogiri (= 1.6.6.2)
   nprogress-rails (= 0.1.6.7)
   paper_trail!
+  parallel
   pdfkit (= 0.6.2)
   pg (= 0.18.2)
   phantomjs (= 1.9.8.0)

app/models/contact.rb

@@ -225,6 +225,10 @@ class Contact < ActiveRecord::Base
       kit.to_pdf
     end
 
+
+    def next_id
+      self.connection.select_value("SELECT nextval('#{self.sequence_name}')")
+    end
   end
 
   def roid

app/models/dnskey.rb

@@ -120,5 +120,9 @@ class Dnskey < ActiveRecord::Base
     def bin_to_hex(s)
       s.each_byte.map { |b| format('%02X', b) }.join
     end
+
+    def next_id
+      self.connection.select_value("SELECT nextval('#{self.sequence_name}')")
+    end
   end
 end

app/models/domain.rb

@@ -221,6 +221,10 @@ class Domain < ActiveRecord::Base
       )
     end
 
+    def next_id
+      self.connection.select_value("SELECT nextval('#{self.sequence_name}')")
+    end
+
     # rubocop: disable Metrics/AbcSize
     # rubocop: disable Metrics/CyclomaticComplexity
     # rubocop: disable Metrics/PerceivedComplexity
@@ -821,8 +825,8 @@ class Domain < ActiveRecord::Base
     log[:admin_contacts] = admin_contact_ids
     log[:tech_contacts]  = tech_contact_ids
     log[:nameservers]    = nameserver_ids
+    log[:dnskeys]        = dnskey_ids
     log[:registrant]     = [registrant_id]
-    log[:domain_statuses] = domain_status_ids
     log
   end
 

app/models/epp/domain.rb

@@ -698,6 +698,9 @@ class Epp::Domain < Domain
       end
 
       attach_legal_document(self.class.parse_legal_document_from_frame(frame))
+
+      # we want to transfer data to new owner at any case.
+      # We also hope that if domain is not valid, new registrar would be better.
       save!(validate: false)
 
       return dt

app/models/legacy/contact.rb

@@ -1,5 +1,12 @@
 module Legacy
   class Contact < Db
+    IDENT_TYPE_MAP = {
+        2 => ::Contact::PRIV,
+        3 => ::Contact::PASSPORT,
+        4 => ::Contact::ORG,
+        6 => ::Contact::BIRTHDAY
+    }
+
     self.table_name = :contact
     belongs_to :object_registry, foreign_key: :id
     belongs_to :object, foreign_key: :id

app/models/legacy/contact_history.rb

@@ -0,0 +1,66 @@
+module Legacy
+  class ContactHistory < Db
+    self.table_name = :contact_history
+    self.primary_key = :id
+
+    belongs_to :object_registry, foreign_key: :id
+    belongs_to :object, foreign_key: :id
+    belongs_to :contact, foreign_key: :id
+    belongs_to :history, foreign_key: :historyid
+    has_one :object_history, foreign_key: :historyid, primary_key: :historyid
+
+    def get_current_contact_object(time, change_param)
+      x = self
+      if 4 == x.ssntype
+        name = x.organization.try(:strip).presence || x.name.try(:strip).presence
+      else
+        name = x.name.try(:strip).presence || x.organization.try(:strip).presence
+      end
+
+      {
+        code: x.object_registry.name.try(:strip),
+        phone: x.telephone.try(:strip),
+        email: [x.email.try(:strip), x.notifyemail.try(:strip)].uniq.select(&:present?).join(', '),
+        fax: x.fax.try(:strip),
+        created_at: x.object_registry.try(:crdate),
+        updated_at: x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update),
+        ident: x.ssn.try(:strip),
+        ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype],
+        auth_info: x.object_history.authinfopw.try(:strip),
+        name: name,
+        registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id),
+        creator_str: x.object_registry.try(:registrar).try(:name),
+        updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name),
+        legacy_id: x.id,
+        street: [x.street1.try(:strip), x.street2.try(:strip), x.street3.try(:strip)].compact.join(", "),
+        city: x.city.try(:strip),
+        zip: x.postalcode.try(:strip),
+        state: x.stateorprovince.try(:strip),
+        country_code: x.country.try(:strip),
+        statuses: ::Legacy::ObjectState.states_for_contact_at(x.id, time)
+      }
+    end
+
+    class << self
+      def changes_dates_for domain_id
+        sql = %Q{SELECT  dh.historyid, valid_from, valid_to
+              FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};}
+
+        hash = {}
+        find_by_sql(sql).each do |rec|
+          hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from
+          hash[rec.valid_to.try(:to_time)]   = [{id: rec.historyid, klass: self, param: :valid_to}]   if rec.valid_to
+        end
+        hash
+      end
+
+      def get_record_at domain_id, rec_id
+        sql = %Q{SELECT  dh.*, h.valid_from, h.valid_to
+            from contact_history dh JOIN history h ON dh.historyid=h.id
+            where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;}
+        find_by_sql(sql).first
+      end
+
+    end
+  end
+end

app/models/legacy/dnskey.rb

@@ -1,8 +1,10 @@
 module Legacy
   class Dnskey < Db
     self.table_name = :dnskey
+    self.primary_key = :id
 
     belongs_to :object_registry, foreign_key: :id
     belongs_to :object, foreign_key: :id
+    has_one :object_history, foreign_key: :historyid, primary_key: :historyid
   end
 end

app/models/legacy/dnskey_history.rb

@@ -0,0 +1,53 @@
+module Legacy
+  class DnskeyHistory < Db
+    self.table_name = :dnskey_history
+
+    belongs_to :object_registry, foreign_key: :id
+    belongs_to :object, foreign_key: :id
+    has_one :object_history, foreign_key: :historyid, primary_key: :historyid
+
+
+    def self.for_at(keysetid, time)
+      return [] unless keysetid
+
+      sql = %Q{select distinct dh.id, dh.keysetid, dh.flags, dh.protocol, dh.alg, dh.key,
+                  first_value(history.valid_from) OVER (PARTITION BY key ORDER BY history.valid_from ASC NULLS FIRST) valid_from,
+                  first_value(history.valid_to) OVER (PARTITION BY key ORDER BY history.valid_to DESC NULLS FIRST) valid_to
+                FROM dnskey_history dh JOIN history ON dh.historyid=history.id
+                WHERE dh.keysetid IN (#{keysetid})
+                AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+                AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)
+                ORDER BY dh.id;}
+      find_by_sql(sql)
+    end
+
+    def new_object_hash(old_domain, new_domain)
+      new_object_mains(new_domain).merge(
+          creator_str: old_domain.object_registry.try(:registrar).try(:name),
+          updator_str: old_domain.object_history.try(:registrar).try(:name) || old_domain.object_registry.try(:registrar).try(:name),
+          legacy_domain_id: old_domain.id,
+          legacy_keyset_id: keysetid,
+          updated_at: (!object_registry.try(:object_history) || object_registry.try(:object_history).read_attribute(:update).nil?) ? (try(:crdate)||Time.zone.now) : object_registry.try(:object_history).read_attribute(:update)
+      )
+    end
+
+    def new_object_mains(new_domain)
+      @new_object_mains ||= {
+       domain_id:  new_domain.id,
+       flags:      flags,
+       protocol:   protocol,
+       alg:        alg,
+       public_key: key
+      }
+    end
+
+    def historical_data(old_domain, new_domain, time_attr = :valid_from)
+      {
+          whodunnit: old_domain.user.try(:id),
+          object: nil,
+          object_changes: new_object_hash(old_domain, new_domain).each_with_object({}){|(k,v), h| h[k] = [nil, v]},
+          created_at: [try(time_attr), old_domain.try(time_attr)].max
+      }
+    end
+  end
+end

app/models/legacy/domain.rb

@@ -7,10 +7,34 @@ module Legacy
     belongs_to :nsset, foreign_key: :nsset
     # belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact'
 
-    has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id
+    has_many :object_states, foreign_key: :object_id
     has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset
     has_many :domain_contact_maps, foreign_key: :domainid
     has_many :nsset_contact_maps, foreign_key: :nssetid, primary_key: :nsset
     has_many :domain_histories, foreign_key: :id
+    alias_method :history, :domain_histories
+
+
+    def new_states
+      domain_statuses = []
+      object_states.valid.each do |state|
+        next if state.name.blank?
+        domain_statuses << state.name
+      end
+
+      # OK status is default
+      domain_statuses << DomainStatus::OK if domain_statuses.empty?
+    end
+
+    def self.new_registrar_cached old_id
+      @new_registrar_cache ||= {}
+      @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id)
+    end
+
+    def self.new_api_user_cached old_id
+      @new_api_user_cache ||= {}
+      @new_api_user_cache[old_id] ||= Legacy::Domain.new_registrar_cached(old_id).try(:api_users).try(:first)
+    end
+
   end
 end

app/models/legacy/domain_contact_map_history.rb

@@ -0,0 +1,5 @@
+module Legacy
+  class DomainContactMapHistory < Db
+    self.table_name = :domain_contact_map_history
+  end
+end

app/models/legacy/domain_history.rb

@@ -1,7 +1,235 @@
 module Legacy
   class DomainHistory < Db
     self.table_name = :domain_history
+    self.primary_key = :id
+    class_attribute :dnssecs
+    class_attribute :namesrvs
 
+    belongs_to :object_registry, foreign_key: :id
+    belongs_to :object, foreign_key: :id
     belongs_to :domain, foreign_key: :id
+    belongs_to :history, foreign_key: :historyid
+    has_one :object_history, foreign_key: :historyid, primary_key: :historyid
+    has_many :nsset_histories,  foreign_key: :id, primary_key: :nsset
+    has_many :domain_contact_map_histories, foreign_key: :historyid, primary_key: :historyid
+    has_many :nsset_contact_map_histories,  foreign_key: :historyid, primary_key: :historyid
+
+    def get_current_domain_object(time, change_param)
+      x = self
+      {
+          name:          SimpleIDN.to_unicode(x.object_registry.name.try(:strip)),
+          registrar_id:  ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id),
+          registrant_id: new_registrant_id,
+          registered_at: x.object_registry.try(:crdate),
+          valid_from:    x.object_registry.try(:crdate),
+          valid_to:      x.exdate,
+          auth_info:     x.object_history.authinfopw.try(:strip),
+          created_at:    x.object_registry.try(:crdate),
+          updated_at:    x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update),
+          name_dirty:    x.object_registry.name.try(:strip),
+          name_puny:     SimpleIDN.to_ascii(x.object_registry.name.try(:strip)),
+          period:        1,
+          period_unit:   'y',
+          creator_str:   x.object_registry.try(:registrar).try(:name),
+          updator_str:   x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name),
+          legacy_id:     x.id,
+          legacy_registrar_id:  x.object_history.try(:clid),
+          legacy_registrant_id: x.registrant,
+          statuses:      Legacy::ObjectState.states_for_domain_at(x.id, time)
+      }
+    end
+
+    def get_admin_contact_new_ids
+      c_ids = domain_contact_map_histories.pluck(:contactid).join("','")
+      DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id)
+    end
+    def get_tech_contact_new_ids
+      c_ids = nsset_contact_map_histories.pluck(:contactid).join("','")
+      DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id)
+    end
+
+    def new_registrant_id
+      @new_registrant_id ||= ::Contact.find_by(legacy_id: registrant).try(:id)
+    end
+
+    def user
+      @user ||= begin
+        obj_his = Legacy::ObjectHistory.find_by(historyid: historyid)
+        Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid)
+      end
+    end
+
+    def history_domain
+      self
+    end
+
+
+    # returns imported nameserver ids
+    def import_nameservers_history(new_domain, time)
+      self.class.namesrvs ||= {}
+      self.class.namesrvs[id] ||= {}
+      ids = []
+
+      nsset_histories.at(time).to_a.each do |nsset|
+        nsset.host_histories.at(time).each do |host|
+          ips = {ipv4: [],ipv6: []}
+          host.host_ipaddr_map_histories.where.not(ipaddr: nil).at(time).each do |ip_map|
+            ips[:ipv4] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv4?
+            ips[:ipv6] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv6?
+          end
+
+          main_attrs = {
+              hostname: SimpleIDN.to_unicode(host.fqdn.try(:strip)),
+              ipv4: ips[:ipv4].sort,
+              ipv6: ips[:ipv6].sort,
+              legacy_domain_id:  id,
+              domain_id: new_domain.id,
+          }
+          server = main_attrs.merge(
+              creator_str: object_registry.try(:registrar).try(:name),
+              updator_str: object_history.try(:registrar).try(:name) || object_registry.try(:registrar).try(:name),
+              created_at: nsset.object_registry.try(:crdate),
+              updated_at: nsset.object_registry.try(:object_history).read_attribute(:update) || nsset.object_registry.try(:crdate)
+          )
+
+
+          if val = self.class.namesrvs[id][main_attrs]
+            ids << val
+          else # if not found we should check current dnssec and historical if changes were done
+            # firstly we need to select the first historical object to take the earliest from create or destroy
+            if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").
+                where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").
+                where("object->>'hostname'='#{main_attrs[:hostname]}'").
+                reorder("created_at ASC").first
+              server[:id] = version.item_id.to_i
+              version.item.versions.where(event: :create).first_or_create!(
+                  whodunnit: user.try(:id),
+                  object: nil,
+                  object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
+                  created_at: time
+              )
+              if !version.ipv4.sort.eql?(main_attrs[:ipv4]) || !version.ipv6.sort.eql?(main_attrs[:ipv6])
+                object_changes = {}
+                server.stringify_keys.each{|k, v| object_changes[k] = [v, version.object[k]] if v != version.object[k] }
+                version.item.versions.where(event: :update).create!(
+                    whodunnit: user.try(:id),
+                    object: server,
+                    object_changes: object_changes,
+                    created_at: time
+                )
+              end
+
+            # if no historical data - try to load existing
+            elsif (list = ::Nameserver.where(domain_id: main_attrs[:domain_id], legacy_domain_id: main_attrs[:legacy_domain_id], hostname: main_attrs[:hostname]).to_a).any?
+              if new_no_version = list.detect{|e|e.versions.where(event: :create).none?} # no create version, so was created via import
+                server[:id] = new_no_version.id.to_i
+                new_no_version.versions.where(event: :create).first_or_create!(
+                    whodunnit: user.try(:id),
+                    object: nil,
+                    object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
+                    created_at: time
+                )
+                if !new_no_version.ipv4.sort.eql?(main_attrs[:ipv4]) || !new_no_version.ipv6.sort.eql?(main_attrs[:ipv6])
+                  object_changes = {}
+                  server.stringify_keys.each{|k, v| object_changes[k] = [v, new_no_version.attributes[k]] if v != new_no_version.attributes[k] }
+                  new_no_version.versions.where(event: :update).create!(
+                      whodunnit: user.try(:id),
+                      object: server,
+                      object_changes: object_changes,
+                      created_at: time
+                  )
+                end
+              else
+                server[:id] = ::Nameserver.next_id
+                create_nameserver_history(server,time)
+              end
+
+            else
+              server[:id] = ::Nameserver.next_id
+              create_nameserver_history(server,time)
+            end
+            self.class.namesrvs[id][main_attrs] = server[:id]
+            ids << server[:id]
+
+          end
+        end
+
+      end
+      ids
+    end
+
+    def create_nameserver_history server, time
+      ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :create).first_or_create!(
+          whodunnit: user.try(:id), object: nil, created_at: time,
+          object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
+      )
+      ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :destroy).create!(
+          whodunnit: user.try(:id), object: server, created_at: Time.now + 2.days,
+          object_changes: {},
+      )
+    end
+
+    # returns imported dnskey ids
+    def import_dnskeys_history(new_domain, time)
+      self.class.dnssecs ||= {}
+      self.class.dnssecs[id] ||= {}
+      ids = []
+      Legacy::DnskeyHistory.for_at(keyset, time).each do |dns|
+        # checking if we have create history for dnskey (cache)
+        if val = self.class.dnssecs[id][dns]
+          ids << val
+        else # if not found we should check current dnssec and historical if changes were done
+          # if current object wasn't changed
+          if item=::Dnskey.where(dns.new_object_mains(new_domain)).first
+            item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain))
+            self.class.dnssecs[id][dns] = item.id
+            ids << item.id
+          # if current object was changed
+          elsif (versions = ::DnskeyVersion.where("object->>'legacy_domain_id'='#{id}'").to_a).any?
+            versions.each do |v|
+              if v.object.slice(*dns.new_object_mains(new_domain).stringify_keys.keys) == dns.new_object_mains(new_domain).keys
+                self.class.dnssecs[id][dns] = v.item_id
+                ids << v.item_id
+                v.item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain))
+              end
+            end
+          # if no history was here
+          else
+            item=::Dnskey.new(id: ::Dnskey.next_id)
+            DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :create).first_or_create!(dns.historical_data(self, new_domain))
+            DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :destroy).first_or_create!(dns.historical_data(self, new_domain), :valid_to) if dns.valid_to
+            self.class.dnssecs[id][dns] = item.id
+            ids << item.id
+          end
+        end
+      end
+
+      ids
+    end
+
+
+    class << self
+      def changes_dates_for domain_id
+        sql = %Q{SELECT  dh.*, valid_from, valid_to
+              FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};}
+
+        hash = {}
+        find_by_sql(sql).each do |rec|
+          hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from
+          hash[rec.valid_to.try(:to_time)]   = [{id: rec.historyid, klass: self, param: :valid_to}]   if rec.valid_to
+        end
+        hash
+      end
+
+      def get_record_at domain_id, rec_id
+        sql = %Q{SELECT  dh.*, h.valid_from, h.valid_to
+            from domain_history dh JOIN history h ON dh.historyid=h.id
+            where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;}
+        find_by_sql(sql).first
+      end
+
+
+
+    end
   end
 end

app/models/legacy/history.rb

@@ -0,0 +1,5 @@
+module Legacy
+  class History < Db
+    self.table_name = :history
+  end
+end

app/models/legacy/host_history.rb

@@ -0,0 +1,15 @@
+module Legacy
+  class HostHistory < Db
+    self.table_name = :host_history
+    self.primary_key = :id
+
+    belongs_to :history, foreign_key: :historyid
+    has_many :host_ipaddr_maps, foreign_key: :hostid
+    has_many :host_ipaddr_map_histories,  foreign_key: :hostid, primary_key: :id
+
+    def self.at(time)
+      joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+            AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)")
+    end
+  end
+end

app/models/legacy/host_ipaddr_map_history.rb

@@ -0,0 +1,12 @@
+module Legacy
+  class HostIpaddrMapHistory < Db
+    self.table_name = :host_ipaddr_map_history
+    self.primary_key = :id
+    belongs_to :history, foreign_key: :historyid
+
+    def self.at(time)
+      joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+            AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)")
+    end
+  end
+end

app/models/legacy/nsset_contact_map_history.rb

@@ -0,0 +1,5 @@
+module Legacy
+  class NssetContactMapHistory < Db
+    self.table_name = :nsset_contact_map_history
+  end
+end

app/models/legacy/nsset_history.rb

@@ -0,0 +1,17 @@
+module Legacy
+  class NssetHistory < Db
+    self.table_name = :nsset_history
+    self.primary_key = :id
+
+    belongs_to :object, foreign_key: :id
+    belongs_to :object_registry, foreign_key: :id
+    belongs_to :history, foreign_key: :historyid, primary_key: :id
+    has_many :hosts, foreign_key: :nssetid
+    has_many :host_histories, foreign_key: :nssetid, primary_key: :id
+
+    def self.at(time)
+      joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+            AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)")
+    end
+  end
+end

app/models/legacy/object_state.rb

@@ -1,6 +1,9 @@
 module Legacy
   class ObjectState < Db
     self.table_name = :object_state
+    attr_accessor :history_domain
+
+    scope :valid, -> { where('valid_to IS NULL') }
 
     # legacy values. Just for log
     # 2 => "serverRenewProhibited",
@@ -77,5 +80,81 @@ module Legacy
 
       map[state_id]
     end
+
+    def get_current_domain_object(time, param)
+      d_his = Legacy::DomainHistory.get_record_at(object_id, historyid)
+      @history_domain = d_his
+
+      hash  = d_his.get_current_domain_object(time, param)
+      hash[:statuses] = Legacy::ObjectState.states_for_domain_at(object_id, time + 1)
+
+      hash
+    end
+
+    def get_current_contact_object(time, param)
+      d_his = Legacy::ContactHistory.get_record_at(object_id, historyid)
+      hash  = d_his.get_current_contact_object(time, param)
+      hash[:statuses] = Legacy::ObjectState.states_for_contact_at(object_id, time + 1)
+
+      hash
+    end
+
+    class << self
+       def changes_dates_for domain_id
+         sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.valid_from, state.valid_to,
+                    extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix
+                    FROM object_history t_2
+                      JOIN object_state state ON (t_2.historyid >= state.ohid_from
+                                                  AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))
+                                                 AND t_2.id = state.object_id
+                    WHERE state.object_id=#{domain_id};}
+         hash = {}
+         find_by_sql(sql).each do |rec|
+           hash[rec.valid_from.try(:to_time)] = [{id: rec.state_dot_id, klass: self, param: :valid_from}] if rec.valid_from
+           hash[rec.valid_to.try(:to_time)]   = [{id: rec.state_dot_id, klass: self, param: :valid_to}]   if rec.valid_to
+         end
+         hash
+       end
+
+      def get_record_at domain_id, rec_id
+        sql = %Q{SELECT distinct t_2.historyid, state.*
+                    FROM object_history t_2
+                      JOIN object_state state ON (t_2.historyid >= state.ohid_from
+                                                  AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))
+                                                 AND t_2.id = state.object_id
+                    WHERE state.object_id=#{domain_id} AND  state.id = #{rec_id};}
+        find_by_sql(sql).first
+      end
+
+      def states_for_domain_at(domain_id, time)
+        sql = %Q{SELECT state.state_id
+          FROM object_history t_2
+            JOIN object_state state ON (t_2.historyid >= state.ohid_from
+                                        AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))
+                                       AND t_2.id = state.object_id
+          WHERE state.object_id=#{domain_id}
+            AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+            AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)
+          }
+        arr = find_by_sql(sql).uniq
+        arr.map!(&:name) if arr.any?
+        arr.present? ? arr : [::DomainStatus::OK]
+      end
+
+
+      def states_for_contact_at(contact_id, time)
+        sql = %Q{SELECT state.state_id
+          FROM object_history t_2
+            JOIN object_state state ON (t_2.historyid >= state.ohid_from
+                                        AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))
+                                       AND t_2.id = state.object_id
+          WHERE state.object_id=#{contact_id}
+            AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ)
+            AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)
+          }
+
+        (find_by_sql(sql).uniq.to_a.map(&:name) + [::Contact::OK]).compact.uniq
+      end
+    end
   end
 end

app/models/nameserver.rb

@@ -98,5 +98,9 @@ class Nameserver < ActiveRecord::Base
       # ignoring ips
       rel
     end
+
+    def next_id
+      self.connection.select_value("SELECT nextval('#{self.sequence_name}')")
+    end
   end
 end

config/application-example.yml

@@ -37,6 +37,7 @@ app_name: '.EE Registry'
 zonefile_export_dir: 'export/zonefiles'
 bank_statement_import_dir: 'import/bank_statements'
 legal_documents_dir: 'import/legal_documents'
+legacy_legal_documents_dir: 'import/legacy_legal_documents'
 time_zone: 'Tallinn' # more zones by rake time:zones:all
 
 openssl_config_path: '/etc/ssl/openssl.cnf'

db/migrate/20160106101725_version_object_is_jsonb.rb

@@ -0,0 +1,26 @@
+class VersionObjectIsJsonb < ActiveRecord::Migration
+  def up
+    change_column :log_contacts, :object, :jsonb, using: "object::jsonb"
+    execute %q(CREATE  INDEX  "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int)))
+    change_column :log_domains, :object, :jsonb, using: "object::jsonb"
+    execute %q(CREATE  INDEX  "log_domains_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int)))
+
+    change_column :log_dnskeys, :object, :jsonb, using: "object::jsonb"
+    execute %q(CREATE  INDEX  "log_dnskeys_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int)))
+    change_column :log_nameservers, :object, :jsonb, using: "object::jsonb"
+    execute %q(CREATE  INDEX  "log_nameservers_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int)))
+
+    add_index :registrars, :legacy_id rescue true
+  end
+  def down
+    change_column :log_contacts,    :object, :json, using: "object::json"
+    change_column :log_domains,     :object, :json, using: "object::json"
+    change_column :log_dnskeys,     :object, :json, using: "object::json"
+    change_column :log_nameservers, :object, :json, using: "object::json"
+
+    drop_index :log_contacts_object_legacy_id
+    drop_index :log_domains_object_legacy_id
+    drop_index :log_dnskeys_object_legacy_id
+    drop_index :log_nameservers_object_legacy_id
+  end
+end

lib/tasks/import.rake

@@ -364,16 +364,6 @@ namespace :import do
       legacy_contact_id
     )
 
-    # rubocop: disable Lint/UselessAssignment
-    domain_status_columns = %w(
-      description
-      value
-      creator_str
-      updator_str
-      legacy_domain_id
-    )
-    # rubocop: enable Lint/UselessAssignment
-
     nameserver_columns = %w(
       hostname
       ipv4
@@ -398,7 +388,6 @@ namespace :import do
 
     domains, nameservers, dnskeys, domain_contacts = [], [], [], []
     existing_domain_ids = Domain.pluck(:legacy_id)
-    user = "rake-#{`whoami`.strip} #{ARGV.join ' '}"
     count = 0
 
     Legacy::Domain.includes(
@@ -414,16 +403,6 @@ namespace :import do
       count += 1
 
       begin
-        # domain statuses
-        domain_statuses = []
-        x.object_states.each do |state|
-          next if state.name.blank?
-          domain_statuses << state.name
-        end
-
-        # OK status is default
-        domain_statuses << DomainStatus::OK if domain_statuses.empty?
-
         domains << [
           x.object_registry.name.try(:strip),
           Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id),
@@ -442,7 +421,7 @@ namespace :import do
           x.id,
           x.object_registry.try(:crid),
           x.registrant,
-          domain_statuses
+          x.new_states
         ]
 
         # admin contacts
@@ -774,6 +753,7 @@ namespace :import do
 
     puts "-----> Imported zones in #{(Time.zone.now.to_f - start).round(2)} seconds"
   end
+
 end
 
 def parse_zone_ns_data(domain, zone)

lib/tasks/import_history.rake

@@ -0,0 +1,195 @@
+namespace :import do
+  desc 'Import all history'
+  task history_all: :environment do
+    Rake::Task['import:history_contacts'].invoke
+    Rake::Task['import:history_domains'].invoke
+  end
+
+  def parallel_import all_ids
+    thread_pool = (Parallel.processor_count rescue 4) - 1
+    threads     = []
+
+    all_ids.each_with_index do |one_id, i|
+      process = Process.fork do
+        begin
+          yield(one_id, i)
+        rescue => e
+          Rails.logger.error("[EXCEPTION] #{Process.pid}")
+          Rails.logger.error("#{Process.pid} #{e.message}" )
+          Rails.logger.error("#{Process.pid} #{e.backtrace.join("\n")}")
+        ensure
+          ActiveRecord::Base.remove_connection
+          Process.exit!
+        end
+      end
+
+      threads << process
+      if threads.count >= thread_pool
+        threads.delete(Process.wait(0))
+      end
+    end
+
+    Process.waitall
+  end
+
+
+  desc 'Import contact history'
+  task history_contacts: :environment do
+    old_ids  = Legacy::ContactHistory.uniq.pluck(:id)
+    old_size = old_ids.size
+    parallel_import(old_ids) do |legacy_contact_id, process_idx|
+      start = Time.now.to_f
+      Contact.transaction do
+        data      = []
+        contact   = Contact.find_by(legacy_id: legacy_contact_id)
+        version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first
+        contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact
+        contact ||= Contact.new(id: ::Contact.next_id, legacy_id: legacy_contact_id)
+        next if contact.versions.where(event: :create).any?
+        # add here to skip domains whith create history
+
+        # 1. add domain changes
+        # 2. add states
+        # compose hash of change time -> Object changes
+        last_changes = nil
+        history  = Legacy::ObjectState.changes_dates_for(legacy_contact_id)
+        con_his  = Legacy::ContactHistory.changes_dates_for(legacy_contact_id)
+        last_contact_action = con_his.sort.last[1].last # need to identify if we delete
+
+        # merging changes together
+        con_his.each do |time, klasses|
+          if history.has_key?(time)
+            history[time] = history[time] | klasses
+          else
+            history[time] = klasses
+          end
+        end
+
+        keys = history.keys.compact.sort
+        i = 0
+        keys.each_with_index do |time|
+          history[time].each do |orig_history_klass|
+            changes   = {}
+            responder = orig_history_klass[:klass].get_record_at(legacy_contact_id, orig_history_klass[:id])
+            new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param])
+            new_attrs[:id] = contact.id
+
+            event = :update
+            event = :create  if i == 0
+            if orig_history_klass == last_contact_action && responder.valid_to.present?
+              event = :destroy
+              new_attrs = {}
+            end
+
+            new_attrs.each do |k, v|
+              if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end
+            end
+            next if changes.blank? && event != :destroy
+            obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid)
+            user    = Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid)
+
+            hash = {
+                item_type: Contact.to_s,
+                item_id:   contact.id,
+                event:     event,
+                whodunnit: user.try(:id),
+                object:    last_changes,
+                object_changes: changes,
+                created_at: time
+            }
+            data << hash
+
+            last_changes = new_attrs
+            i += 1
+          end
+        end
+        ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any?
+      end
+      puts "[PID: #{Process.pid}] Legacy Contact #{legacy_contact_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}"
+    end
+  end
+
+
+
+  desc 'Import domain history'
+  task history_domains: :environment do
+    old_ids  = Legacy::DomainHistory.uniq.pluck(:id)
+    old_size = old_ids.size
+    parallel_import(old_ids) do |legacy_domain_id, process_idx|
+      start = Time.now.to_f
+      Domain.transaction do
+        domain   = Domain.find_by(legacy_id: legacy_domain_id)
+        version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first
+        domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain
+        domain ||= Domain.new(id: ::Domain.next_id, legacy_id: legacy_domain_id)
+        next if domain.versions.where(event: :create).any?
+        # add here to skip domains whith create history
+
+        # 1. add domain changes
+        # 2. add states
+        # compose hash of change time -> Object changes
+        last_changes = nil
+        history  = Legacy::ObjectState.changes_dates_for(legacy_domain_id)
+        dom_his  = Legacy::DomainHistory.changes_dates_for(legacy_domain_id)
+        last_domain_action = dom_his.sort.last[1].last # need to identify if we delete
+
+        # merging changes together
+        dom_his.each do |time, klasses|
+          if history.has_key?(time)
+            history[time] = history[time] | klasses
+          else
+            history[time] = klasses
+          end
+        end
+
+        keys = history.keys.compact.sort
+        i = 0
+        keys.each_with_index do |time|
+          history[time].each do |orig_history_klass|
+            changes   = {}
+            responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id])
+            new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param])
+            new_attrs[:id]         = domain.id
+            new_attrs[:updated_at] = time
+
+            event = :update
+            event = :create  if i == 0
+            if orig_history_klass == last_domain_action && responder.valid_to.present?
+              event = :destroy
+              new_attrs = {}
+            end
+
+            new_attrs.each do |k, v|
+              if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end
+            end
+            next if changes.blank? && event != :destroy
+
+            DomainVersion.create!(
+                item_type: domain.class,
+                item_id:   domain.id,
+                event:     event,
+                whodunnit: responder.history_domain.user.try(:id),
+                object:    last_changes,
+                object_changes: changes,
+                created_at: time,
+                children: {
+                    admin_contacts: responder.history_domain.get_admin_contact_new_ids,
+                    tech_contacts:  responder.history_domain.get_tech_contact_new_ids,
+                    nameservers:    responder.history_domain.import_nameservers_history(domain, time),
+                    dnskeys:        responder.history_domain.import_dnskeys_history(domain, time),
+                    registrant:     [responder.history_domain.new_registrant_id]
+                }
+            )
+
+            last_changes = new_attrs
+            i += 1
+          end
+        end
+      end
+      puts "[PID: #{Process.pid}] Legacy Domain #{legacy_domain_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}"
+    end
+  end
+
+
+
+end