Story#108602614 - optimize contacts history import

app/models/legacy/contact_history.rb

@@ -28,7 +28,7 @@ module Legacy
         ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype],
         auth_info: x.object_history.authinfopw.try(:strip),
         name: name,
-        registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id),
+        registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id),
         creator_str: x.object_registry.try(:registrar).try(:name),
         updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name),
         legacy_id: x.id,
@@ -43,13 +43,13 @@ module Legacy
 
     class << self
       def changes_dates_for domain_id
-        sql = %Q{SELECT  dh.*, valid_from
+        sql = %Q{SELECT  dh.historyid, valid_from, valid_to
               FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};}
-        # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]}
 
         hash = {}
         find_by_sql(sql).each do |rec|
           hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from
+          hash[rec.valid_to.try(:to_time)]   = [{id: rec.historyid, klass: self, param: :valid_to}]   if rec.valid_to
         end
         hash
       end

app/models/legacy/domain.rb

@@ -25,5 +25,10 @@ module Legacy
       # OK status is default
       domain_statuses << DomainStatus::OK if domain_statuses.empty?
     end
+
+    def self.new_registrar_cached old_id
+      @new_registrar_cache ||= {}
+      @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id)
+    end
   end
 end

app/models/legacy/domain_history.rb

@@ -18,7 +18,7 @@ module Legacy
       x = self
       {
           name:          SimpleIDN.to_unicode(x.object_registry.name.try(:strip)),
-          registrar_id:  ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id),
+          registrar_id:  ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id),
           registrant_id: new_registrant_id,
           registered_at: x.object_registry.try(:crdate),
           valid_from:    x.object_registry.try(:crdate),
@@ -55,7 +55,7 @@ module Legacy
     def user
       @user ||= begin
         obj_his = Legacy::ObjectHistory.find_by(historyid: historyid)
-        Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first)
+        Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first)
       end
     end
 
@@ -97,7 +97,10 @@ module Legacy
             ids << val
           else # if not found we should check current dnssec and historical if changes were done
             # firstly we need to select the first historical object to take the earliest from create or destroy
-            if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").where("object->>'hostname'='#{main_attrs[:hostname]}'").reorder("created_at ASC").first
+            if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").
+                where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").
+                where("object->>'hostname'='#{main_attrs[:hostname]}'").
+                reorder("created_at ASC").first
               server[:id] = version.item_id.to_i
               version.item.versions.where(event: :create).first_or_create!(
                   whodunnit: user.try(:id),
@@ -209,11 +212,11 @@ module Legacy
       def changes_dates_for domain_id
         sql = %Q{SELECT  dh.*, valid_from
               FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};}
-        # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]}
 
         hash = {}
         find_by_sql(sql).each do |rec|
           hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from
+          hash[rec.valid_to.try(:to_time)]   = [{id: rec.historyid, klass: self, param: :valid_to}]   if rec.valid_to
         end
         hash
       end

app/models/legacy/object_state.rb

@@ -101,7 +101,7 @@ module Legacy
 
     class << self
        def changes_dates_for domain_id
-         sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.*,
+         sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.valid_from, state.valid_to,
                     extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix
                     FROM object_history t_2
                       JOIN object_state state ON (t_2.historyid >= state.ohid_from
@@ -127,7 +127,7 @@ module Legacy
       end
 
       def states_for_domain_at(domain_id, time)
-        sql = %Q{SELECT state.*
+        sql = %Q{SELECT state.state_id
           FROM object_history t_2
             JOIN object_state state ON (t_2.historyid >= state.ohid_from
                                         AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))
@@ -143,7 +143,7 @@ module Legacy
 
 
       def states_for_contact_at(contact_id, time)
-        sql = %Q{SELECT state.*
+        sql = %Q{SELECT state.state_id
           FROM object_history t_2
             JOIN object_state state ON (t_2.historyid >= state.ohid_from
                                         AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL))

db/migrate/20160106101725_version_object_is_jsonb.rb

@@ -0,0 +1,10 @@
+class VersionObjectIsJsonb < ActiveRecord::Migration
+  def up
+    change_column :log_contacts, :object, :jsonb, using: "object::jsonb"
+    execute %q(CREATE  INDEX  "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int)))
+    add_index :registrars, :legacy_id
+  end
+  def down
+    change_column :log_contacts, :object, :json, using: "object::json"
+  end
+end

lib/tasks/import_history.rake

@@ -8,8 +8,10 @@ namespace :import do
 
   desc 'Import contact history'
   task history_contacts: :environment do
-    Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id|
+    Legacy::ContactHistory.uniq.where("id > 4175").pluck(:id).each do |legacy_contact_id|
+      start = Time.now.to_f
       Contact.transaction do
+        data      = []
         contact   = Contact.find_by(legacy_id: legacy_contact_id)
         version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first
         contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact
@@ -55,7 +57,7 @@ namespace :import do
             end
             next if changes.blank? && event != :destroy
             obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid)
-            user    = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first)
+            user    = Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first)
 
             hash = {
                 item_type: Contact.to_s,
@@ -66,13 +68,15 @@ namespace :import do
                 object_changes: changes,
                 created_at: time
             }
-            ContactVersion.create!(hash)
+            data << hash
 
             last_changes = new_attrs
             i += 1
           end
         end
+        ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any?
       end
+      puts "Legacy Contact id #{legacy_contact_id} finished in #{Time.now.to_f - start}"
     end
   end
 
@@ -115,7 +119,6 @@ namespace :import do
             new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param])
             new_attrs[:id]         = domain.id
             new_attrs[:updated_at] = time
-            p time
 
             event = :update
             event = :create  if i == 0