Story#108521790 - nameservers/domains import optimization + more save nameserver history creation if object had already manipulations

app/models/legacy/domain_history.rb

@@ -69,6 +69,7 @@ module Legacy
       self.class.namesrvs ||= {}
       self.class.namesrvs[id] ||= {}
       ids = []
+      to_import = []
 
       nsset_histories.at(time).to_a.each do |nsset|
         nsset.host_histories.at(time).each do |host|
@@ -102,42 +103,42 @@ module Legacy
                 where("object->>'hostname'='#{main_attrs[:hostname]}'").
                 reorder("created_at ASC").first
               server[:id] = version.item_id.to_i
-              version.item.versions.where(event: :create).first_or_create!(
-                  whodunnit: user.try(:id),
-                  object: nil,
-                  object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
-                  created_at: time
-              )
+              ::NameserverVersion.where(item_type: ::NameserverVersion, item_id: version.item_id).where(event: :create).first_or_create!(
+                    whodunnit: user.try(:id),
+                    object: nil,
+                    object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
+                    created_at: time
+                )
               if !version.object["ipv4"].sort.eql?(main_attrs[:ipv4]) || !version.object["ipv6"].sort.eql?(main_attrs[:ipv6])
                 object_changes = {}
                 server.stringify_keys.each{|k, v| object_changes[k] = [v, version.object[k]] if v != version.object[k] }
-                version.item.versions.where(event: :update).create!(
+                to_import << version.item.versions.where(event: :update).build(
                     whodunnit: user.try(:id),
                     object: server,
                     object_changes: object_changes,
                     created_at: time
-                )
+                ).attributes
               end
 
             # if no historical data - try to load existing
             elsif (list = ::Nameserver.where(domain_id: main_attrs[:domain_id], legacy_domain_id: main_attrs[:legacy_domain_id], hostname: main_attrs[:hostname]).to_a).any?
               if new_no_version = list.detect{|e|e.versions.where(event: :create).none?} # no create version, so was created via import
                 server[:id] = new_no_version.id.to_i
-                new_no_version.versions.where(event: :create).first_or_create!(
+                to_import << new_no_version.versions.where(event: :create).build(
                     whodunnit: user.try(:id),
                     object: nil,
                     object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]},
                     created_at: time
-                )
+                ).attributes
                 if !new_no_version.ipv4.sort.eql?(main_attrs[:ipv4]) || !new_no_version.ipv6.sort.eql?(main_attrs[:ipv6])
                   object_changes = {}
                   server.stringify_keys.each{|k, v| object_changes[k] = [v, new_no_version.attributes[k]] if v != new_no_version.attributes[k] }
-                  new_no_version.versions.where(event: :update).create!(
+                  to_import << new_no_version.versions.where(event: :update).build(
                       whodunnit: user.try(:id),
                       object: server,
                       object_changes: object_changes,
                       created_at: time
-                  )
+                  ).attributes
                 end
               else
                 server[:id] = ::Nameserver.next_id
@@ -153,6 +154,7 @@ module Legacy
 
           end
         end
+        ::NameserverVersion.import_without_validations_or_callbacks to_import.first.keys, to_import.map(&:values) if to_import.any?
 
       end
       ids
@@ -197,7 +199,7 @@ module Legacy
           else
             item=::Dnskey.new(id: ::Dnskey.next_id)
             DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :create).first_or_create!(dns.historical_data(self, new_domain))
-            DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :destroy).first_or_create!(dns.historical_data(self, new_domain), :valid_to) if dns.valid_to
+            DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :destroy).first_or_create!(dns.historical_data(self, new_domain, :valid_to)) if dns.valid_to
             self.class.dnssecs[id][dns] = item.id
             ids << item.id
           end

lib/tasks/import_history.rake

@@ -118,6 +118,7 @@ namespace :import do
     parallel_import(old_ids) do |legacy_domain_id, process_idx|
       start = Time.now.to_f
       Domain.transaction do
+        data     = []
         domain   = Domain.find_by(legacy_id: legacy_domain_id)
         version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first
         domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain
@@ -164,7 +165,7 @@ namespace :import do
             end
             next if changes.blank? && event != :destroy
 
-            DomainVersion.create!(
+            hash = {
                 item_type: domain.class,
                 item_id:   domain.id,
                 event:     event,
@@ -179,12 +180,14 @@ namespace :import do
                     dnskeys:        responder.history_domain.import_dnskeys_history(domain, time),
                     registrant:     [responder.history_domain.new_registrant_id]
                 }
-            )
+            }
+            data << hash
 
             last_changes = new_attrs
             i += 1
           end
         end
+        DomainVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any?
       end
       puts "[PID: #{Process.pid}] Legacy Domain #{legacy_domain_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}"
     end