From 1fb0259806191c9ac9d34f94c248b9f54d369870 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 6 Jan 2016 15:18:30 +0200 Subject: [PATCH] Story#108602614 - optimize contacts history import --- app/models/legacy/contact_history.rb | 6 +++--- app/models/legacy/domain.rb | 5 +++++ app/models/legacy/domain_history.rb | 11 +++++++---- app/models/legacy/object_state.rb | 6 +++--- db/migrate/20160106101725_version_object_is_jsonb.rb | 10 ++++++++++ lib/tasks/import_history.rake | 11 +++++++---- 6 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 db/migrate/20160106101725_version_object_is_jsonb.rb diff --git a/app/models/legacy/contact_history.rb b/app/models/legacy/contact_history.rb index 040ba0dec..41f664d6e 100644 --- a/app/models/legacy/contact_history.rb +++ b/app/models/legacy/contact_history.rb @@ -28,7 +28,7 @@ module Legacy ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype], auth_info: x.object_history.authinfopw.try(:strip), name: name, - registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), creator_str: x.object_registry.try(:registrar).try(:name), updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), legacy_id: x.id, @@ -43,13 +43,13 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT dh.*, valid_from + sql = %Q{SELECT dh.historyid, valid_from, valid_to FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} - # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} hash = {} find_by_sql(sql).each do |rec| hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to end hash end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index 5725c3142..949a14702 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -25,5 +25,10 @@ module Legacy # OK status is default domain_statuses << DomainStatus::OK if domain_statuses.empty? end + + def self.new_registrar_cached old_id + @new_registrar_cache ||= {} + @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id) + end end end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 8d84eeea0..4b75b81da 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -18,7 +18,7 @@ module Legacy x = self { name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), - registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), registrant_id: new_registrant_id, registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), @@ -55,7 +55,7 @@ module Legacy def user @user ||= begin obj_his = Legacy::ObjectHistory.find_by(historyid: historyid) - Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) end end @@ -97,7 +97,10 @@ module Legacy ids << val else # if not found we should check current dnssec and historical if changes were done # firstly we need to select the first historical object to take the earliest from create or destroy - if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").where("object->>'hostname'='#{main_attrs[:hostname]}'").reorder("created_at ASC").first + if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'"). + where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'"). + where("object->>'hostname'='#{main_attrs[:hostname]}'"). + reorder("created_at ASC").first server[:id] = version.item_id.to_i version.item.versions.where(event: :create).first_or_create!( whodunnit: user.try(:id), @@ -209,11 +212,11 @@ module Legacy def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} - # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} hash = {} find_by_sql(sql).each do |rec| hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to end hash end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index df892866d..31ea6b408 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -101,7 +101,7 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.*, + sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.valid_from, state.valid_to, extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from @@ -127,7 +127,7 @@ module Legacy end def states_for_domain_at(domain_id, time) - sql = %Q{SELECT state.* + sql = %Q{SELECT state.state_id FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) @@ -143,7 +143,7 @@ module Legacy def states_for_contact_at(contact_id, time) - sql = %Q{SELECT state.* + sql = %Q{SELECT state.state_id FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) diff --git a/db/migrate/20160106101725_version_object_is_jsonb.rb b/db/migrate/20160106101725_version_object_is_jsonb.rb new file mode 100644 index 000000000..68212d2b6 --- /dev/null +++ b/db/migrate/20160106101725_version_object_is_jsonb.rb @@ -0,0 +1,10 @@ +class VersionObjectIsJsonb < ActiveRecord::Migration + def up + change_column :log_contacts, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) + add_index :registrars, :legacy_id + end + def down + change_column :log_contacts, :object, :json, using: "object::json" + end +end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index aab50c177..35ab7ff07 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -8,8 +8,10 @@ namespace :import do desc 'Import contact history' task history_contacts: :environment do - Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| + Legacy::ContactHistory.uniq.where("id > 4175").pluck(:id).each do |legacy_contact_id| + start = Time.now.to_f Contact.transaction do + data = [] contact = Contact.find_by(legacy_id: legacy_contact_id) version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact @@ -55,7 +57,7 @@ namespace :import do end next if changes.blank? && event != :destroy obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + user = Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) hash = { item_type: Contact.to_s, @@ -66,13 +68,15 @@ namespace :import do object_changes: changes, created_at: time } - ContactVersion.create!(hash) + data << hash last_changes = new_attrs i += 1 end end + ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any? end + puts "Legacy Contact id #{legacy_contact_id} finished in #{Time.now.to_f - start}" end end @@ -115,7 +119,6 @@ namespace :import do new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) new_attrs[:id] = domain.id new_attrs[:updated_at] = time - p time event = :update event = :create if i == 0