From e11c1b5d20a7bff4d8be035be5d76b73b6e3e591 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Tue, 17 Nov 2015 12:17:26 +0200 Subject: [PATCH 01/31] Prepare history import --- app/models/legacy/domain.rb | 1 + app/models/legacy/domain_history.rb | 2 ++ app/models/legacy/history.rb | 5 +++++ lib/tasks/import.rake | 24 ++++++++++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 app/models/legacy/history.rb diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index d0a05d178..64506b1ef 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -12,5 +12,6 @@ module Legacy has_many :domain_contact_maps, foreign_key: :domainid has_many :nsset_contact_maps, foreign_key: :nssetid, primary_key: :nsset has_many :domain_histories, foreign_key: :id + alias_method :history, :domain_histories end end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 234585ac3..08ab52836 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -3,5 +3,7 @@ module Legacy self.table_name = :domain_history belongs_to :domain, foreign_key: :id + belongs_to :history, foreign_key: :historyid + has_one :object_history, foreign_key: :historyid, primary_key: :historyid end end diff --git a/app/models/legacy/history.rb b/app/models/legacy/history.rb new file mode 100644 index 000000000..98ef464b0 --- /dev/null +++ b/app/models/legacy/history.rb @@ -0,0 +1,5 @@ +module Legacy + class History < Db + self.table_name = :history + end +end \ No newline at end of file diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 28786d278..519a3279c 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -774,6 +774,30 @@ namespace :import do puts "-----> Imported zones in #{(Time.zone.now.to_f - start).round(2)} seconds" end + + desc 'Import history' + task history: :environment do + Domain.where.not(legacy_id: nil).find_each do |domain| + next if domain.versions.where(action: :create).any? + + history = Legacy::DomainHistory.where(id: domain.legacy_id).order("valid_from ASC").to_a + history.each_with_index do |his, i| + event = :update + event = :create if i == 0 + event = :destroy if i + 1 == history.size && his.history.valid_to.present? + + { + item_type: domain.class, + item_id: domain.id, + event: event, + whodunnit: Registrar.find_by(legacy_id: his.object_history.upid || his.object_history.clid), + object: {}, + object_changes: {"id" => [nil, 1111] }, + created_at: his.object_history.try(:update), + } + end + end + end end def parse_zone_ns_data(domain, zone) From 3c0e04beac19d8fa63945023308ce147403243da Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 20 Nov 2015 14:43:34 +0200 Subject: [PATCH 02/31] Story#104525318 - cleanup import code --- app/models/legacy/domain.rb | 14 ++++++++++- app/models/legacy/object_state.rb | 2 ++ lib/tasks/import.rake | 42 +++++++++++++++---------------- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index 64506b1ef..5725c3142 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -7,11 +7,23 @@ module Legacy belongs_to :nsset, foreign_key: :nsset # belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' - has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id + has_many :object_states, foreign_key: :object_id has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset has_many :domain_contact_maps, foreign_key: :domainid has_many :nsset_contact_maps, foreign_key: :nssetid, primary_key: :nsset has_many :domain_histories, foreign_key: :id alias_method :history, :domain_histories + + + def new_states + domain_statuses = [] + object_states.valid.each do |state| + next if state.name.blank? + domain_statuses << state.name + end + + # OK status is default + domain_statuses << DomainStatus::OK if domain_statuses.empty? + end end end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 379d4d175..8ab059d8d 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -2,6 +2,8 @@ module Legacy class ObjectState < Db self.table_name = :object_state + scope :valid, -> { where('valid_to IS NULL') } + # legacy values. Just for log # 2 => "serverRenewProhibited", # 5 => "serverOutzoneManual", diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 519a3279c..d2e0a8b7b 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -364,16 +364,6 @@ namespace :import do legacy_contact_id ) - # rubocop: disable Lint/UselessAssignment - domain_status_columns = %w( - description - value - creator_str - updator_str - legacy_domain_id - ) - # rubocop: enable Lint/UselessAssignment - nameserver_columns = %w( hostname ipv4 @@ -398,7 +388,6 @@ namespace :import do domains, nameservers, dnskeys, domain_contacts = [], [], [], [] existing_domain_ids = Domain.pluck(:legacy_id) - user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 Legacy::Domain.includes( @@ -414,16 +403,6 @@ namespace :import do count += 1 begin - # domain statuses - domain_statuses = [] - x.object_states.each do |state| - next if state.name.blank? - domain_statuses << state.name - end - - # OK status is default - domain_statuses << DomainStatus::OK if domain_statuses.empty? - domains << [ x.object_registry.name.try(:strip), Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), @@ -442,7 +421,7 @@ namespace :import do x.id, x.object_registry.try(:crid), x.registrant, - domain_statuses + x.new_states ] # admin contacts @@ -777,6 +756,25 @@ namespace :import do desc 'Import history' task history: :environment do + # {"id"=>83215, "name"=>"gssb-dsf0pf.ee", "registrar_id"=>17, + # "registered_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, + # "status"=>nil, "valid_from"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, + # "valid_to"=>Fri, 18 Sep 2015 00:00:00 EEST +03:00, "registrant_id"=>262841, + # "auth_info"=>"authinfopw", "created_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, + # "updated_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, "name_dirty"=>"gssb-dsf0pf.ee", + # "name_puny"=>"gssb-dsf0pf.ee", "period"=>1, "period_unit"=>"y", "creator_str"=>"Elkdata OÜ", + # "updator_str"=>"Elkdata OÜ", "legacy_id"=>778106, "legacy_registrar_id"=>13, "legacy_registrant_id"=>778104, + # "outzone_at"=>nil, "delete_at"=>nil, "registrant_verification_asked_at"=>nil, + # "registrant_verification_token"=>nil, + # "pending_json"=>{}, "force_delete_at"=>nil, "statuses"=>["ok"], + # "reserved"=>false, "status_notes"=>{}, "statuses_backup"=>[]} + + Legacy::DomainHistory.uniq.pluck(:id).each do |legacy_domain_id| + # 1. add domain changes + # 2. add states + + end + Domain.where.not(legacy_id: nil).find_each do |domain| next if domain.versions.where(action: :create).any? From df6e254cc6eb836ed05fe9497238740a85fc5cbc Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Mon, 23 Nov 2015 10:11:58 +0200 Subject: [PATCH 03/31] Story#104525318 - get states change timestamps --- app/models/legacy/object_state.rb | 14 ++++++++++++++ lib/tasks/import.rake | 2 ++ 2 files changed, 16 insertions(+) diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 8ab059d8d..9c7938c51 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -79,5 +79,19 @@ module Legacy map[state_id] end + + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT t_2.id, state.* + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id} + ORDER BY t_2.historyid;} + find_by_sql(sql).map{|e| e.attributes.values_at("valid_from", "valid_to") }.flatten.each_with_object({}){|e,h|h[e] = [self]} + end + end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index d2e0a8b7b..0e790bf20 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -772,6 +772,8 @@ namespace :import do Legacy::DomainHistory.uniq.pluck(:id).each do |legacy_domain_id| # 1. add domain changes # 2. add states + # compose hash of change time -> Object changes + history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) end From 3e8309f81824d80965429c94f17d4f7e3bae12b9 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Tue, 24 Nov 2015 16:55:17 +0200 Subject: [PATCH 04/31] Story#104525318 - in domain history import match change records in historical order --- app/models/legacy/domain_history.rb | 28 ++++++++++++ app/models/legacy/object_state.rb | 42 +++++++++++++----- lib/tasks/import.rake | 66 ++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 30 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 08ab52836..d45aa5f92 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -5,5 +5,33 @@ module Legacy belongs_to :domain, foreign_key: :id belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid + + def get_current_domain_object(param) + p "not implemented #{__method__}" + end + + def get_current_changes(param) + p "not implemented #{__method__}" + end + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT dh.*, valid_from--, extract(epoch from h.valid_from) valid_from_unix, extract(epoch from h.valid_to) valid_to_unix + FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} + # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} + + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + end + hash + end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT dh.*, h.valid_from, h.valid_to from domain_history dh JOIN history h ON dh.historyid=h.id + where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} + find_by_sql(sql).first + end + end end end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 9c7938c51..4197276b9 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -80,18 +80,40 @@ module Legacy map[state_id] end + def get_current_domain_object(param) + p "not implemented #{__method__}" + end - class << self + def get_current_changes(param) + p "not implemented #{__method__}" + end + + class << self def changes_dates_for domain_id - sql = %Q{SELECT t_2.id, state.* - FROM object_history t_2 - JOIN object_state state ON (t_2.historyid >= state.ohid_from - AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) - AND t_2.id = state.object_id - WHERE state.object_id=#{domain_id} - ORDER BY t_2.historyid;} - find_by_sql(sql).map{|e| e.attributes.values_at("valid_from", "valid_to") }.flatten.each_with_object({}){|e,h|h[e] = [self]} + sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.*, + extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id};} + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.state_dot_id, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.state_dot_id, klass: self, param: :valid_to}] if rec.valid_to + end + hash end - end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT distinct t_2.id, state.* + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id} AND state.id = #{rec_id};} + find_by_sql(sql).first + end + end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 0e790bf20..abe3c2918 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -769,34 +769,60 @@ namespace :import do # "pending_json"=>{}, "force_delete_at"=>nil, "statuses"=>["ok"], # "reserved"=>false, "status_notes"=>{}, "statuses_backup"=>[]} - Legacy::DomainHistory.uniq.pluck(:id).each do |legacy_domain_id| + Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| + # add here to skip domains whith create history + # 1. add domain changes # 2. add states # compose hash of change time -> Object changes - history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) + history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) + p history.keys + p Legacy::DomainHistory.changes_dates_for(legacy_domain_id).keys + Legacy::DomainHistory.changes_dates_for(legacy_domain_id).each do |time, klasses| + if history.has_key?(time) + history[time] = history[time] | klasses + else + history[time] = klasses + end + end - end + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| - Domain.where.not(legacy_id: nil).find_each do |domain| - next if domain.versions.where(action: :create).any? + p time + history[time].each do |orig_history_klass| + event = :update + event = :create if i == 0 + responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) + responder.get_current_domain_object(orig_history_klass[:param]) + responder.get_current_changes(orig_history_klass[:param]) - history = Legacy::DomainHistory.where(id: domain.legacy_id).order("valid_from ASC").to_a - history.each_with_index do |his, i| - event = :update - event = :create if i == 0 - event = :destroy if i + 1 == history.size && his.history.valid_to.present? - - { - item_type: domain.class, - item_id: domain.id, - event: event, - whodunnit: Registrar.find_by(legacy_id: his.object_history.upid || his.object_history.clid), - object: {}, - object_changes: {"id" => [nil, 1111] }, - created_at: his.object_history.try(:update), - } + i += 1 + end end end + + # Domain.where.not(legacy_id: nil).find_each do |domain| + # next if domain.versions.where(action: :create).any? + # + # history = Legacy::DomainHistory.where(id: domain.legacy_id).order("valid_from ASC").to_a + # history.each_with_index do |his, i| + # event = :update + # event = :create if i == 0 + # event = :destroy if i + 1 == history.size && his.history.valid_to.present? + # + # { + # item_type: domain.class, + # item_id: domain.id, + # event: event, + # whodunnit: Registrar.find_by(legacy_id: his.object_history.upid || his.object_history.clid), + # object: {}, + # object_changes: {"id" => [nil, 1111] }, + # created_at: his.object_history.try(:update), + # } + # end + # end end end From f5a3e51e69143ab6e4e0df087ceb607b5b4cbf43 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 4 Dec 2015 12:19:50 +0200 Subject: [PATCH 05/31] Story#108521790 first scratch to get domain real historical object --- app/models/legacy/domain_history.rb | 32 +++++++++++++++++++++++++---- lib/tasks/import.rake | 2 -- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index d45aa5f92..e5439655d 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -1,13 +1,36 @@ module Legacy class DomainHistory < Db self.table_name = :domain_history + self.primary_key = :id + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id belongs_to :domain, foreign_key: :id belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid - def get_current_domain_object(param) - p "not implemented #{__method__}" + def get_current_domain_object(change_param) + x = self + { + name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), + registrar_id: Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), + registered_at: x.object_registry.try(:crdate), + valid_from: x.object_registry.try(:crdate), + valid_to: x.exdate, + auth_info: x.object.authinfopw.try(:strip), + created_at: x.object_registry.try(:crdate), + updated_at: x.object.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object.read_attribute(:update), + name_dirty: x.object_registry.name.try(:strip), + name_puny: SimpleIDN.to_ascii(x.object_registry.name.try(:strip)), + period: 1, + period_unit: 'y', + creator_str: x.object_registry.try(:registrar).try(:name), + updator_str: x.object.try(:registrar).try(:name) ? x.object.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), + legacy_id: x.id, + legacy_registrar_id: x.object_registry.try(:crid), + legacy_registrant_id: x.registrant, + statuses: x.states + } end def get_current_changes(param) @@ -16,7 +39,7 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT dh.*, valid_from--, extract(epoch from h.valid_from) valid_from_unix, extract(epoch from h.valid_to) valid_to_unix + sql = %Q{SELECT dh.*, valid_from FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} @@ -28,7 +51,8 @@ module Legacy end def get_record_at domain_id, rec_id - sql = %Q{SELECT dh.*, h.valid_from, h.valid_to from domain_history dh JOIN history h ON dh.historyid=h.id + sql = %Q{SELECT dh.*, h.valid_from, h.valid_to + from domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} find_by_sql(sql).first end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index abe3c2918..63130dde9 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -789,8 +789,6 @@ namespace :import do keys = history.keys.compact.sort i = 0 keys.each_with_index do |time| - - p time history[time].each do |orig_history_klass| event = :update event = :create if i == 0 From beee0a0ad8450fb6cd2ec8aeb25e34d312d8bfbd Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 4 Dec 2015 15:41:42 +0200 Subject: [PATCH 06/31] Story#108521790 Domain historical attributes attributes at particular moment --- app/models/legacy/domain_history.rb | 10 +++------- app/models/legacy/object_state.rb | 27 +++++++++++++++++++++------ lib/tasks/import.rake | 5 +++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index e5439655d..380af3907 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -9,11 +9,11 @@ module Legacy belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid - def get_current_domain_object(change_param) + def get_current_domain_object(time, change_param) x = self { name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), - registrar_id: Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), + registrar_id: ::Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), valid_to: x.exdate, @@ -29,14 +29,10 @@ module Legacy legacy_id: x.id, legacy_registrar_id: x.object_registry.try(:crid), legacy_registrant_id: x.registrant, - statuses: x.states + statuses: Legacy::ObjectState.states_for_domain_at(x.id, time) } end - def get_current_changes(param) - p "not implemented #{__method__}" - end - class << self def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 4197276b9..f6f89da0b 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -80,12 +80,12 @@ module Legacy map[state_id] end - def get_current_domain_object(param) - p "not implemented #{__method__}" - end + def get_current_domain_object(time, param) + d_his = Legacy::DomainHistory.get_record_at(object_id, historyid) + hash = d_his.get_current_domain_object(time, param) + hash[:statuses] = Legacy::ObjectState.states_for_domain_at(object_id, time + 1) - def get_current_changes(param) - p "not implemented #{__method__}" + hash end class << self @@ -106,7 +106,7 @@ module Legacy end def get_record_at domain_id, rec_id - sql = %Q{SELECT distinct t_2.id, state.* + sql = %Q{SELECT distinct t_2.historyid, state.* FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) @@ -114,6 +114,21 @@ module Legacy WHERE state.object_id=#{domain_id} AND state.id = #{rec_id};} find_by_sql(sql).first end + + def states_for_domain_at(domain_id, time) + sql = %Q{SELECT state.* + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id} + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + } + arr = find_by_sql(sql).uniq + arr.map!(&:name) if arr.any? + arr.present? ? arr : [DomainStatus::OK] + end end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 63130dde9..f1deeb9a1 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -770,11 +770,13 @@ namespace :import do # "reserved"=>false, "status_notes"=>{}, "statuses_backup"=>[]} Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| + next if Domain.find_by(legacy_id: legacy_domain_id).versions.where(event: :create).any? # add here to skip domains whith create history # 1. add domain changes # 2. add states # compose hash of change time -> Object changes + last_changes = nil history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) p history.keys p Legacy::DomainHistory.changes_dates_for(legacy_domain_id).keys @@ -793,8 +795,7 @@ namespace :import do event = :update event = :create if i == 0 responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) - responder.get_current_domain_object(orig_history_klass[:param]) - responder.get_current_changes(orig_history_klass[:param]) + p responder.get_current_domain_object(time, orig_history_klass[:param]) i += 1 end From 5b273f0df277e403d2e5ecd30b976aaa25b8ea85 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 4 Dec 2015 17:43:49 +0200 Subject: [PATCH 07/31] Story#108521790 Domain save it's changes history to paper_trail --- app/models/legacy/domain_history.rb | 13 +++++-- lib/tasks/import.rake | 53 ++++++++++++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 380af3907..8216fe81f 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -13,7 +13,8 @@ module Legacy x = self { name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), - registrar_id: ::Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), + registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + registrant_id: ::Contact.find_by(legacy_id: x.registrant).try(:id), registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), valid_to: x.exdate, @@ -27,7 +28,7 @@ module Legacy creator_str: x.object_registry.try(:registrar).try(:name), updator_str: x.object.try(:registrar).try(:name) ? x.object.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), legacy_id: x.id, - legacy_registrar_id: x.object_registry.try(:crid), + legacy_registrar_id: x.object_history.try(:clid), legacy_registrant_id: x.registrant, statuses: Legacy::ObjectState.states_for_domain_at(x.id, time) } @@ -52,6 +53,14 @@ module Legacy where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} find_by_sql(sql).first end + + + # def last_history_action domain_id + # sql = %Q{SELECT dh.*, h.valid_from, h.valid_to + # from domain_history dh JOIN history h ON dh.historyid=h.id + # where dh.id=#{domain_id} order by dh.historyid desc limit 1;} + # find_by_sql(sql).first + # end end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index f1deeb9a1..3af559ea0 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -755,20 +755,7 @@ namespace :import do end desc 'Import history' - task history: :environment do - # {"id"=>83215, "name"=>"gssb-dsf0pf.ee", "registrar_id"=>17, - # "registered_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, - # "status"=>nil, "valid_from"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, - # "valid_to"=>Fri, 18 Sep 2015 00:00:00 EEST +03:00, "registrant_id"=>262841, - # "auth_info"=>"authinfopw", "created_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, - # "updated_at"=>Thu, 18 Sep 2014 10:17:13 EEST +03:00, "name_dirty"=>"gssb-dsf0pf.ee", - # "name_puny"=>"gssb-dsf0pf.ee", "period"=>1, "period_unit"=>"y", "creator_str"=>"Elkdata OÜ", - # "updator_str"=>"Elkdata OÜ", "legacy_id"=>778106, "legacy_registrar_id"=>13, "legacy_registrant_id"=>778104, - # "outzone_at"=>nil, "delete_at"=>nil, "registrant_verification_asked_at"=>nil, - # "registrant_verification_token"=>nil, - # "pending_json"=>{}, "force_delete_at"=>nil, "statuses"=>["ok"], - # "reserved"=>false, "status_notes"=>{}, "statuses_backup"=>[]} - + task history_domains: :environment do Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| next if Domain.find_by(legacy_id: legacy_domain_id).versions.where(event: :create).any? # add here to skip domains whith create history @@ -777,10 +764,13 @@ namespace :import do # 2. add states # compose hash of change time -> Object changes last_changes = nil + domain = Domain.find_by(legacy_id: legacy_domain_id) history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) - p history.keys - p Legacy::DomainHistory.changes_dates_for(legacy_domain_id).keys - Legacy::DomainHistory.changes_dates_for(legacy_domain_id).each do |time, klasses| + dom_his = Legacy::DomainHistory.changes_dates_for(legacy_domain_id) + last_domain_action = dom_his.sort.last[1].last # need to identify if we delete + + # merging changes together + dom_his.each do |time, klasses| if history.has_key?(time) history[time] = history[time] | klasses else @@ -792,11 +782,36 @@ namespace :import do i = 0 keys.each_with_index do |time| history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) + new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) + event = :update event = :create if i == 0 - responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) - p responder.get_current_domain_object(time, orig_history_klass[:param]) + if orig_history_klass == last_domain_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? + obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + + DomainVersion.create!( + item_type: domain.class, + item_id: domain.id, + event: event, + whodunnit: user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time, + children: {} + ) + + last_changes = new_attrs i += 1 end end From 057e699272334b4b81450a0f4e8a3cbb0a6bfb1a Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 4 Dec 2015 17:49:51 +0200 Subject: [PATCH 08/31] Add comment to code --- app/models/epp/domain.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/epp/domain.rb b/app/models/epp/domain.rb index 40b374003..12eeaf70d 100644 --- a/app/models/epp/domain.rb +++ b/app/models/epp/domain.rb @@ -691,6 +691,9 @@ class Epp::Domain < Domain end attach_legal_document(self.class.parse_legal_document_from_frame(frame)) + + # we want to transfer data to new owner at any case. + # We also hope that if domain is not valid, new registrar would be better. save!(validate: false) return dt From 91da4db2cead65748f5878e7b28d7f083b6d448e Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 9 Dec 2015 11:46:33 +0200 Subject: [PATCH 09/31] Story#108521790 - rm old comment how to import history --- lib/tasks/import.rake | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 3af559ea0..94b6b8288 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -817,26 +817,6 @@ namespace :import do end end - # Domain.where.not(legacy_id: nil).find_each do |domain| - # next if domain.versions.where(action: :create).any? - # - # history = Legacy::DomainHistory.where(id: domain.legacy_id).order("valid_from ASC").to_a - # history.each_with_index do |his, i| - # event = :update - # event = :create if i == 0 - # event = :destroy if i + 1 == history.size && his.history.valid_to.present? - # - # { - # item_type: domain.class, - # item_id: domain.id, - # event: event, - # whodunnit: Registrar.find_by(legacy_id: his.object_history.upid || his.object_history.clid), - # object: {}, - # object_changes: {"id" => [nil, 1111] }, - # created_at: his.object_history.try(:update), - # } - # end - # end end end From af055aaac5c708b34f6dcecf9523839ef63eab4d Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 9 Dec 2015 17:26:00 +0200 Subject: [PATCH 10/31] Story#108602614 - Import contact history --- app/models/contact.rb | 4 + app/models/legacy/contact.rb | 7 ++ app/models/legacy/contact_history.rb | 66 +++++++++++++ app/models/legacy/object_state.rb | 25 ++++- lib/tasks/import.rake | 64 ------------ lib/tasks/import_history.rake | 141 +++++++++++++++++++++++++++ 6 files changed, 242 insertions(+), 65 deletions(-) create mode 100644 app/models/legacy/contact_history.rb create mode 100644 lib/tasks/import_history.rake diff --git a/app/models/contact.rb b/app/models/contact.rb index 96365d056..9a4bf7bbd 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -219,6 +219,10 @@ class Contact < ActiveRecord::Base kit.to_pdf end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end def roid diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 04f8c34a4..fca3a2b23 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -1,5 +1,12 @@ module Legacy class Contact < Db + IDENT_TYPE_MAP = { + 2 => ::Contact::PRIV, + 3 => ::Contact::PASSPORT, + 4 => ::Contact::ORG, + 6 => ::Contact::BIRTHDAY + } + self.table_name = :contact belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id diff --git a/app/models/legacy/contact_history.rb b/app/models/legacy/contact_history.rb new file mode 100644 index 000000000..040ba0dec --- /dev/null +++ b/app/models/legacy/contact_history.rb @@ -0,0 +1,66 @@ +module Legacy + class ContactHistory < Db + self.table_name = :contact_history + self.primary_key = :id + + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id + belongs_to :contact, foreign_key: :id + belongs_to :history, foreign_key: :historyid + has_one :object_history, foreign_key: :historyid, primary_key: :historyid + + def get_current_contact_object(time, change_param) + x = self + if 4 == x.ssntype + name = x.organization.try(:strip).presence || x.name.try(:strip).presence + else + name = x.name.try(:strip).presence || x.organization.try(:strip).presence + end + + { + code: x.object_registry.name.try(:strip), + phone: x.telephone.try(:strip), + email: [x.email.try(:strip), x.notifyemail.try(:strip)].uniq.select(&:present?).join(', '), + fax: x.fax.try(:strip), + created_at: x.object_registry.try(:crdate), + updated_at: x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update), + ident: x.ssn.try(:strip), + ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype], + auth_info: x.object_history.authinfopw.try(:strip), + name: name, + registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + creator_str: x.object_registry.try(:registrar).try(:name), + updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), + legacy_id: x.id, + street: [x.street1.try(:strip), x.street2.try(:strip), x.street3.try(:strip)].compact.join(", "), + city: x.city.try(:strip), + zip: x.postalcode.try(:strip), + state: x.stateorprovince.try(:strip), + country_code: x.country.try(:strip), + statuses: ::Legacy::ObjectState.states_for_contact_at(x.id, time) + } + end + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT dh.*, valid_from + FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} + # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} + + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + end + hash + end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT dh.*, h.valid_from, h.valid_to + from contact_history dh JOIN history h ON dh.historyid=h.id + where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} + find_by_sql(sql).first + end + + end + end +end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index f6f89da0b..d39ca3d06 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -88,6 +88,14 @@ module Legacy hash end + def get_current_contact_object(time, param) + d_his = Legacy::ContactHistory.get_record_at(object_id, historyid) + hash = d_his.get_current_contact_object(time, param) + hash[:statuses] = Legacy::ObjectState.states_for_contact_at(object_id, time + 1) + + hash + end + class << self def changes_dates_for domain_id sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.*, @@ -127,7 +135,22 @@ module Legacy } arr = find_by_sql(sql).uniq arr.map!(&:name) if arr.any? - arr.present? ? arr : [DomainStatus::OK] + arr.present? ? arr : [::DomainStatus::OK] + end + + + def states_for_contact_at(contact_id, time) + sql = %Q{SELECT state.* + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{contact_id} + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + } + + (find_by_sql(sql).uniq.to_a.map(&:name) + [::Contact::OK]).compact.uniq end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 94b6b8288..1926237a7 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -754,70 +754,6 @@ namespace :import do puts "-----> Imported zones in #{(Time.zone.now.to_f - start).round(2)} seconds" end - desc 'Import history' - task history_domains: :environment do - Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| - next if Domain.find_by(legacy_id: legacy_domain_id).versions.where(event: :create).any? - # add here to skip domains whith create history - - # 1. add domain changes - # 2. add states - # compose hash of change time -> Object changes - last_changes = nil - domain = Domain.find_by(legacy_id: legacy_domain_id) - history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) - dom_his = Legacy::DomainHistory.changes_dates_for(legacy_domain_id) - last_domain_action = dom_his.sort.last[1].last # need to identify if we delete - - # merging changes together - dom_his.each do |time, klasses| - if history.has_key?(time) - history[time] = history[time] | klasses - else - history[time] = klasses - end - end - - keys = history.keys.compact.sort - i = 0 - keys.each_with_index do |time| - history[time].each do |orig_history_klass| - changes = {} - responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) - new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) - - event = :update - event = :create if i == 0 - if orig_history_klass == last_domain_action && responder.valid_to.present? - event = :destroy - new_attrs = {} - end - - new_attrs.each do |k, v| - if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end - end - next if changes.blank? - obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) - - DomainVersion.create!( - item_type: domain.class, - item_id: domain.id, - event: event, - whodunnit: user.try(:id), - object: last_changes, - object_changes: changes, - created_at: time, - children: {} - ) - - last_changes = new_attrs - i += 1 - end - end - end - - end end def parse_zone_ns_data(domain, zone) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake new file mode 100644 index 000000000..862f50001 --- /dev/null +++ b/lib/tasks/import_history.rake @@ -0,0 +1,141 @@ +namespace :import do + desc 'Import contact history' + task history_contacts: :environment do + Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| + Contact.transaction do + contact = Contact.find_by(legacy_id: legacy_contact_id) + version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").first + contact ||= Contact.new(id: version_contact.object["id"], legacy_id: legacy_contact_id) if version_contact + contact ||= Contact.new(id: ::Contact.next_id, legacy_id: legacy_contact_id) + next if contact.versions.where(event: :create).any? + # add here to skip domains whith create history + + # 1. add domain changes + # 2. add states + # compose hash of change time -> Object changes + last_changes = nil + history = Legacy::ObjectState.changes_dates_for(legacy_contact_id) + con_his = Legacy::ContactHistory.changes_dates_for(legacy_contact_id) + last_contact_action = con_his.sort.last[1].last # need to identify if we delete + + # merging changes together + con_his.each do |time, klasses| + if history.has_key?(time) + history[time] = history[time] | klasses + else + history[time] = klasses + end + end + + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| + history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_contact_id, orig_history_klass[:id]) + new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param]) + + event = :update + event = :create if i == 0 + if orig_history_klass == last_contact_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? && event != :destroy + obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + + ContactVersion.create!( + item_type: contact.class, + item_id: contact.id, + event: event, + whodunnit: user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time, + children: {} + ) + + last_changes = new_attrs + i += 1 + end + end + end + end + end + + + desc 'Import domain history' + task history_domains: :environment do + Domain.transaction do + Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| + next if Domain.find_by(legacy_id: legacy_domain_id).versions.where(event: :create).any? + # add here to skip domains whith create history + + # 1. add domain changes + # 2. add states + # compose hash of change time -> Object changes + last_changes = nil + domain = Domain.find_by(legacy_id: legacy_domain_id) + history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) + dom_his = Legacy::DomainHistory.changes_dates_for(legacy_domain_id) + last_domain_action = dom_his.sort.last[1].last # need to identify if we delete + + # merging changes together + dom_his.each do |time, klasses| + if history.has_key?(time) + history[time] = history[time] | klasses + else + history[time] = klasses + end + end + + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| + history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) + new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) + + event = :update + event = :create if i == 0 + if orig_history_klass == last_domain_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? && event != :destroy + obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + + DomainVersion.create!( + item_type: domain.class, + item_id: domain.id, + event: event, + whodunnit: user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time, + children: {} + ) + + last_changes = new_attrs + i += 1 + end + end + end + end + + end + + + +end \ No newline at end of file From b2dd26d1b9d551ae99bfdff3f6f15b1aa3f58680 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 11:27:56 +0200 Subject: [PATCH 11/31] Story#108602614 - take object id from history from item_id --- lib/tasks/import_history.rake | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 862f50001..22d0b4dea 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -4,8 +4,8 @@ namespace :import do Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| Contact.transaction do contact = Contact.find_by(legacy_id: legacy_contact_id) - version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").first - contact ||= Contact.new(id: version_contact.object["id"], legacy_id: legacy_contact_id) if version_contact + version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first + contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact contact ||= Contact.new(id: ::Contact.next_id, legacy_id: legacy_contact_id) next if contact.versions.where(event: :create).any? # add here to skip domains whith create history @@ -34,6 +34,7 @@ namespace :import do changes = {} responder = orig_history_klass[:klass].get_record_at(legacy_contact_id, orig_history_klass[:id]) new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param]) + new_attrs[:id] = contact.id event = :update event = :create if i == 0 @@ -49,16 +50,16 @@ namespace :import do obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) - ContactVersion.create!( - item_type: contact.class, + hash = { + item_type: Contact.to_s, item_id: contact.id, event: event, whodunnit: user.try(:id), object: last_changes, object_changes: changes, - created_at: time, - children: {} - ) + created_at: time + } + ContactVersion.create!(hash) last_changes = new_attrs i += 1 From 764a8fa0d3ea593679ee165f63e93cd537bc60a5 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 11:50:51 +0200 Subject: [PATCH 12/31] Story#108602614 - apply contacts history import improvements to domains --- app/models/legacy/domain_history.rb | 6 +++--- lib/tasks/import_history.rake | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 8216fe81f..467c470b9 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -18,15 +18,15 @@ module Legacy registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), valid_to: x.exdate, - auth_info: x.object.authinfopw.try(:strip), + auth_info: x.object_history.authinfopw.try(:strip), created_at: x.object_registry.try(:crdate), - updated_at: x.object.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object.read_attribute(:update), + updated_at: x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update), name_dirty: x.object_registry.name.try(:strip), name_puny: SimpleIDN.to_ascii(x.object_registry.name.try(:strip)), period: 1, period_unit: 'y', creator_str: x.object_registry.try(:registrar).try(:name), - updator_str: x.object.try(:registrar).try(:name) ? x.object.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), + updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), legacy_id: x.id, legacy_registrar_id: x.object_history.try(:clid), legacy_registrant_id: x.registrant, diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 22d0b4dea..4f3a79e0f 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -74,14 +74,17 @@ namespace :import do task history_domains: :environment do Domain.transaction do Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| - next if Domain.find_by(legacy_id: legacy_domain_id).versions.where(event: :create).any? + domain = Domain.find_by(legacy_id: legacy_domain_id) + version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first + domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain + domain ||= Domain.new(id: ::Domain.next_id, legacy_id: legacy_domain_id) + next if domain.versions.where(event: :create).any? # add here to skip domains whith create history # 1. add domain changes # 2. add states # compose hash of change time -> Object changes last_changes = nil - domain = Domain.find_by(legacy_id: legacy_domain_id) history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) dom_his = Legacy::DomainHistory.changes_dates_for(legacy_domain_id) last_domain_action = dom_his.sort.last[1].last # need to identify if we delete @@ -102,6 +105,7 @@ namespace :import do changes = {} responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) + new_attrs[:id] = domain.id event = :update event = :create if i == 0 From 33ee6f62564f1f98f0cfd59fe918700367e90218 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 12:12:01 +0200 Subject: [PATCH 13/31] Story#108602614 - add next_id to domain class --- app/models/domain.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/domain.rb b/app/models/domain.rb index 988cd43e3..bb7a8356c 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -221,6 +221,10 @@ class Domain < ActiveRecord::Base ) end + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end + # rubocop: disable Metrics/AbcSize # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity From 8dfb62151d80d3c08c77cbed471e465c768fc007 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 12:52:17 +0200 Subject: [PATCH 14/31] Story#108521790 - add admin contacts relation to domain history --- app/models/legacy/domain_contact_map_history.rb | 5 +++++ app/models/legacy/domain_history.rb | 12 +++++++++++- lib/tasks/import_history.rake | 7 ++++++- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 app/models/legacy/domain_contact_map_history.rb diff --git a/app/models/legacy/domain_contact_map_history.rb b/app/models/legacy/domain_contact_map_history.rb new file mode 100644 index 000000000..9ad3a6f7e --- /dev/null +++ b/app/models/legacy/domain_contact_map_history.rb @@ -0,0 +1,5 @@ +module Legacy + class DomainContactMapHistory < Db + self.table_name = :domain_contact_map_history + end +end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 467c470b9..7a81db413 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -8,13 +8,14 @@ module Legacy belongs_to :domain, foreign_key: :id belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid + has_many :domain_contact_map_histories, foreign_key: :historyid, primary_key: :historyid def get_current_domain_object(time, change_param) x = self { name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), - registrant_id: ::Contact.find_by(legacy_id: x.registrant).try(:id), + registrant_id: new_registrant_id, registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), valid_to: x.exdate, @@ -34,6 +35,15 @@ module Legacy } end + def get_admin_contact_new_ids + c_ids = domain_contact_map_histories.pluck(:contactid).join("','") + DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id) + end + + def new_registrant_id + @new_registrant_id ||= ::Contact.find_by(legacy_id: registrant).try(:id) + end + class << self def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 4f3a79e0f..bf12e3e18 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -129,7 +129,12 @@ namespace :import do object: last_changes, object_changes: changes, created_at: time, - children: {} + children: { + admin_contacts: [responder.get_admin_contact_new_ids], + tech_contacts: [], + nameservers: [], + registrant: [responder.new_registrant_id] + } ) last_changes = new_attrs From 112dac9e7c1f13c42bb28a822b1ad7ee561b21fa Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 13:53:55 +0200 Subject: [PATCH 15/31] Story#108521790 - add tech contacts relation to domain history --- app/models/domain.rb | 2 +- app/models/legacy/domain_history.rb | 5 +++++ app/models/legacy/nsset_contact_map_history.rb | 5 +++++ lib/tasks/import_history.rake | 3 ++- 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 app/models/legacy/nsset_contact_map_history.rb diff --git a/app/models/domain.rb b/app/models/domain.rb index bb7a8356c..a76de9e4a 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -807,8 +807,8 @@ class Domain < ActiveRecord::Base log[:admin_contacts] = admin_contact_ids log[:tech_contacts] = tech_contact_ids log[:nameservers] = nameserver_ids + log[:dnskeys] = dnskey_ids log[:registrant] = [registrant_id] - log[:domain_statuses] = domain_status_ids log end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 7a81db413..d127e15c6 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -9,6 +9,7 @@ module Legacy belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid has_many :domain_contact_map_histories, foreign_key: :historyid, primary_key: :historyid + has_many :nsset_contact_map_histories, foreign_key: :historyid, primary_key: :historyid def get_current_domain_object(time, change_param) x = self @@ -39,6 +40,10 @@ module Legacy c_ids = domain_contact_map_histories.pluck(:contactid).join("','") DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id) end + def get_tech_contact_new_ids + c_ids = nsset_contact_map_histories.pluck(:contactid).join("','") + DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id) + end def new_registrant_id @new_registrant_id ||= ::Contact.find_by(legacy_id: registrant).try(:id) diff --git a/app/models/legacy/nsset_contact_map_history.rb b/app/models/legacy/nsset_contact_map_history.rb new file mode 100644 index 000000000..ffcb74793 --- /dev/null +++ b/app/models/legacy/nsset_contact_map_history.rb @@ -0,0 +1,5 @@ +module Legacy + class NssetContactMapHistory < Db + self.table_name = :nsset_contact_map_history + end +end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index bf12e3e18..ba810b4ae 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -131,8 +131,9 @@ namespace :import do created_at: time, children: { admin_contacts: [responder.get_admin_contact_new_ids], - tech_contacts: [], + tech_contacts: [responder.get_tech_contact_new_ids], nameservers: [], + dnskeys: [], registrant: [responder.new_registrant_id] } ) From aa7bd933e044158a509c5d732cda13b16b1fe15a Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 13:54:30 +0200 Subject: [PATCH 16/31] Story#108521790 - add tech contacts relation to domain history --- lib/tasks/import_history.rake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index ba810b4ae..97a791085 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -130,11 +130,11 @@ namespace :import do object_changes: changes, created_at: time, children: { - admin_contacts: [responder.get_admin_contact_new_ids], - tech_contacts: [responder.get_tech_contact_new_ids], - nameservers: [], - dnskeys: [], - registrant: [responder.new_registrant_id] + admin_contacts: responder.get_admin_contact_new_ids, + tech_contacts: responder.get_tech_contact_new_ids, + nameservers: [], + dnskeys: [], + registrant: [responder.new_registrant_id] } ) From 5f0421af3843b588830fd60b3c85508278345a16 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 13:56:03 +0200 Subject: [PATCH 17/31] Story#108521790 - handling when nested objects has been changed --- lib/tasks/import_history.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 97a791085..0cbe3296c 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -105,7 +105,8 @@ namespace :import do changes = {} responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) - new_attrs[:id] = domain.id + new_attrs[:id] = domain.id + new_attrs[:updated_at] = time event = :update event = :create if i == 0 From f6d16f51e9fc4b4eb66aa480af219ef6c805c0f2 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 10 Dec 2015 17:28:28 +0200 Subject: [PATCH 18/31] Story#108521790 - first try to import nameservers --- app/models/legacy/domain_history.rb | 67 ++++++++++++++++++++ app/models/legacy/host_history.rb | 15 +++++ app/models/legacy/host_ipaddr_map_history.rb | 12 ++++ app/models/legacy/nsset_history.rb | 17 +++++ app/models/nameserver.rb | 4 ++ lib/tasks/import_history.rake | 7 +- 6 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 app/models/legacy/host_history.rb create mode 100644 app/models/legacy/host_ipaddr_map_history.rb create mode 100644 app/models/legacy/nsset_history.rb diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index d127e15c6..b8f47f3f7 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -8,6 +8,7 @@ module Legacy belongs_to :domain, foreign_key: :id belongs_to :history, foreign_key: :historyid has_one :object_history, foreign_key: :historyid, primary_key: :historyid + has_many :nsset_histories, foreign_key: :id, primary_key: :nsset has_many :domain_contact_map_histories, foreign_key: :historyid, primary_key: :historyid has_many :nsset_contact_map_histories, foreign_key: :historyid, primary_key: :historyid @@ -49,6 +50,72 @@ module Legacy @new_registrant_id ||= ::Contact.find_by(legacy_id: registrant).try(:id) end + def user + @user ||= Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + end + + + # returns imported nameserver ids + def import_nameservers_history(new_domain, time) + #removing previous nameservers + NameserverVersion.where("object->>legacy_domain_id").where(event: :create).where("created_at <= ?", time).each do |nv| + if NameserverVersion.where(item_type: nv.item_type, item_id: nv.item_id, event: :destroy).none? + NameserverVersion.create!( + item_type: nv.item_type, + item_id: nv.item_id, + event: :destroy, + whodunnit: user.try(:id), + object: nv.object_changes.each_with_object({}){|(k,v),hash| hash[k] = v.last }, + object_changes: {}, + created_at: time + ) + end + end + + + if (nssets = nsset_histories.at(time).to_a).any? + ids = [] + nssets.each do |nsset| + nsset.host_histories.at(time).each do |host| + ips = {ipv4: [],ipv6: []} + host.host_ipaddr_map_histories.at(time).each do |ip_map| + next unless ip_map.ipaddr + ips[:ipv4] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv4? + ips[:ipv6] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv6? + end + + server = { + id: Nameserver.next_id, + hostname: host.fqdn.try(:strip), + ipv4: ips[:ipv4], + ipv6: ips[:ipv6], + creator_str: object_registry.try(:registrar).try(:name), + updator_str: object_history.try(:registrar).try(:name) || object_registry.try(:registrar).try(:name), + legacy_domain_id: id, + domain_id: new_domain.id, + created_at: nsset.object_registry.try(:crdate), + updated_at: nsset.object_registry.try(:object_history).read_attribute(:update) || nsset.object_registry.try(:crdate) + } + + NameserverVersion.create!( + item_type: Nameserver.to_s, + item_id: server[:id], + event: :create, + whodunnit: user.try(:id), + object: nil, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: time + ) + ids << server[:id] + end + + end + ids + else + [] + end + end + class << self def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from diff --git a/app/models/legacy/host_history.rb b/app/models/legacy/host_history.rb new file mode 100644 index 000000000..437e476af --- /dev/null +++ b/app/models/legacy/host_history.rb @@ -0,0 +1,15 @@ +module Legacy + class HostHistory < Db + self.table_name = :host_history + self.primary_key = :id + + belongs_to :history, foreign_key: :historyid + has_many :host_ipaddr_maps, foreign_key: :hostid + has_many :host_ipaddr_map_histories, foreign_key: :hostid, primary_key: :id + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/legacy/host_ipaddr_map_history.rb b/app/models/legacy/host_ipaddr_map_history.rb new file mode 100644 index 000000000..9f84121f7 --- /dev/null +++ b/app/models/legacy/host_ipaddr_map_history.rb @@ -0,0 +1,12 @@ +module Legacy + class HostIpaddrMapHistory < Db + self.table_name = :host_ipaddr_map_history + self.primary_key = :id + belongs_to :history, foreign_key: :historyid + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/legacy/nsset_history.rb b/app/models/legacy/nsset_history.rb new file mode 100644 index 000000000..1a88ebc12 --- /dev/null +++ b/app/models/legacy/nsset_history.rb @@ -0,0 +1,17 @@ +module Legacy + class NssetHistory < Db + self.table_name = :nsset_history + self.primary_key = :id + + belongs_to :object, foreign_key: :id + belongs_to :object_registry, foreign_key: :id + belongs_to :history, foreign_key: :historyid, primary_key: :id + has_many :hosts, foreign_key: :nssetid + has_many :host_histories, foreign_key: :nssetid, primary_key: :id + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/nameserver.rb b/app/models/nameserver.rb index ccedf0880..245dc602e 100644 --- a/app/models/nameserver.rb +++ b/app/models/nameserver.rb @@ -98,5 +98,9 @@ class Nameserver < ActiveRecord::Base # ignoring ips rel end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 0cbe3296c..3895bfa4e 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -107,6 +107,8 @@ namespace :import do new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) new_attrs[:id] = domain.id new_attrs[:updated_at] = time + p time + p responder.import_nameservers_history(domain, time) if responder.respond_to?(:import_nameservers_history) event = :update event = :create if i == 0 @@ -120,13 +122,12 @@ namespace :import do end next if changes.blank? && event != :destroy obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) - DomainVersion.create!( + p DomainVersion.new( item_type: domain.class, item_id: domain.id, event: event, - whodunnit: user.try(:id), + whodunnit: responder.user.try(:id), object: last_changes, object_changes: changes, created_at: time, From 01a82a2960d6d89bd2ccb12469b1829714a021c1 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 11 Dec 2015 14:36:21 +0200 Subject: [PATCH 19/31] Story#108521790 - update that code works with nameservers --- app/models/legacy/domain_history.rb | 37 +++++++++++++++++------------ app/models/legacy/object_state.rb | 3 +++ lib/tasks/import_history.rake | 13 +++++----- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index b8f47f3f7..5ea18947a 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -51,26 +51,33 @@ module Legacy end def user - @user ||= Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + @user ||= begin + obj_his = Legacy::ObjectHistory.find_by(historyid: historyid) + Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + end + end + + def history_domain + self end # returns imported nameserver ids def import_nameservers_history(new_domain, time) - #removing previous nameservers - NameserverVersion.where("object->>legacy_domain_id").where(event: :create).where("created_at <= ?", time).each do |nv| - if NameserverVersion.where(item_type: nv.item_type, item_id: nv.item_id, event: :destroy).none? - NameserverVersion.create!( - item_type: nv.item_type, - item_id: nv.item_id, - event: :destroy, - whodunnit: user.try(:id), - object: nv.object_changes.each_with_object({}){|(k,v),hash| hash[k] = v.last }, - object_changes: {}, - created_at: time - ) - end - end + # #removing previous nameservers + # NameserverVersion.where("object_changes->>'legacy_domain_id' = '[nil,#{id}]'").where(event: :create).where("created_at <= ?", time).each do |nv| + # if NameserverVersion.where(item_type: nv.item_type, item_id: nv.item_id, event: :destroy).none? + # NameserverVersion.create!( + # item_type: nv.item_type, + # item_id: nv.item_id, + # event: :destroy, + # whodunnit: user.try(:id), + # object: nv.object_changes.each_with_object({}){|(k,v),hash| hash[k] = v.last }, + # object_changes: {}, + # created_at: time + # ) + # end + # end if (nssets = nsset_histories.at(time).to_a).any? diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index d39ca3d06..df892866d 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -1,6 +1,7 @@ module Legacy class ObjectState < Db self.table_name = :object_state + attr_accessor :history_domain scope :valid, -> { where('valid_to IS NULL') } @@ -82,6 +83,8 @@ module Legacy def get_current_domain_object(time, param) d_his = Legacy::DomainHistory.get_record_at(object_id, historyid) + @history_domain = d_his + hash = d_his.get_current_domain_object(time, param) hash[:statuses] = Legacy::ObjectState.states_for_domain_at(object_id, time + 1) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 3895bfa4e..1948e1b88 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -108,7 +108,6 @@ namespace :import do new_attrs[:id] = domain.id new_attrs[:updated_at] = time p time - p responder.import_nameservers_history(domain, time) if responder.respond_to?(:import_nameservers_history) event = :update event = :create if i == 0 @@ -121,22 +120,22 @@ namespace :import do if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end end next if changes.blank? && event != :destroy - obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + responder.import_nameservers_history(domain, time) if responder.respond_to?(:import_nameservers_history) - p DomainVersion.new( + DomainVersion.create!( item_type: domain.class, item_id: domain.id, event: event, - whodunnit: responder.user.try(:id), + whodunnit: responder.history_domain.user.try(:id), object: last_changes, object_changes: changes, created_at: time, children: { - admin_contacts: responder.get_admin_contact_new_ids, - tech_contacts: responder.get_tech_contact_new_ids, + admin_contacts: responder.history_domain.get_admin_contact_new_ids, + tech_contacts: responder.history_domain.get_tech_contact_new_ids, nameservers: [], dnskeys: [], - registrant: [responder.new_registrant_id] + registrant: [responder.history_domain.new_registrant_id] } ) From d41488472aba158797810a9c9f6ac866d4433c67 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 11 Dec 2015 17:43:20 +0200 Subject: [PATCH 20/31] Story#108521790 - import dnskey history --- app/models/dnskey.rb | 4 ++ app/models/legacy/dnskey.rb | 2 + app/models/legacy/dnskey_history.rb | 8 ++++ lib/tasks/import_history.rake | 66 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 app/models/legacy/dnskey_history.rb diff --git a/app/models/dnskey.rb b/app/models/dnskey.rb index 02b43d729..ef8f68a39 100644 --- a/app/models/dnskey.rb +++ b/app/models/dnskey.rb @@ -120,5 +120,9 @@ class Dnskey < ActiveRecord::Base def bin_to_hex(s) s.each_byte.map { |b| format('%02X', b) }.join end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end end diff --git a/app/models/legacy/dnskey.rb b/app/models/legacy/dnskey.rb index 3fa47f48b..782673ef8 100644 --- a/app/models/legacy/dnskey.rb +++ b/app/models/legacy/dnskey.rb @@ -1,8 +1,10 @@ module Legacy class Dnskey < Db self.table_name = :dnskey + self.primary_key = :id belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id + has_one :object_history, foreign_key: :historyid, primary_key: :historyid end end diff --git a/app/models/legacy/dnskey_history.rb b/app/models/legacy/dnskey_history.rb new file mode 100644 index 000000000..1f62b1002 --- /dev/null +++ b/app/models/legacy/dnskey_history.rb @@ -0,0 +1,8 @@ +module Legacy + class DnskeyHistory < Db + self.table_name = :dnskey_history + + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id + end +end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 1948e1b88..b982492cb 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -1,4 +1,12 @@ namespace :import do + desc 'Import all history' + task history_all: :environment do + Rake::Task['import:history_contacts'].invoke + Rake::Task['import:history_dnskeys'].invoke + Rake::Task['import:history_domains'].invoke + end + + desc 'Import contact history' task history_contacts: :environment do Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| @@ -70,6 +78,64 @@ namespace :import do end + desc 'Import contact history' + task history_dnskeys: :environment do + Legacy::DnskeyHistory.uniq.pluck(:id).each do |legacy_dnskey_id| + Dnskey.transaction do + contact = Dnskey.find_by(legacy_id: legacy_dnskey_id) + version_dns = DnskeytVersion.where("object->>'legacy_id' = '#{legacy_dnskey_id}'").select(:item_id).first + contact ||= Dnskey.new(id: version_dns.item_id, legacy_id: legacy_dnskey_id) if version_dns + contact ||= Dnskey.new(id: ::Contact.next_id, legacy_id: legacy_dnskey_id) + next if contact.versions.where(event: :create).any? + # add here to skip domains whith create history + + last_changes = nil + history = Legacy::ContactHistory.changes_dates_for(legacy_dnskey_id) + last_contact_action = history.sort.last[1].last # need to identify if we delete + + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| + history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_dnskey_id, orig_history_klass[:id]) + new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param]) + new_attrs[:id] = contact.id + + event = :update + event = :create if i == 0 + if orig_history_klass == last_contact_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? && event != :destroy + obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + + hash = { + item_type: Dnskey.to_s, + item_id: contact.id, + event: event, + whodunnit: user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time + } + DnskeyVersion.create!(hash) + + last_changes = new_attrs + i += 1 + end + end + end + end + end + + desc 'Import domain history' task history_domains: :environment do Domain.transaction do From ac6eccb24c17b7ebd1958492747efc316405624c Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Tue, 15 Dec 2015 13:39:09 +0200 Subject: [PATCH 21/31] Story 108521790 importing dnskey --- app/models/legacy/dnskey_history.rb | 45 ++++++++++++++++++++++ app/models/legacy/domain_history.rb | 39 +++++++++++++++++++ lib/tasks/import_history.rake | 60 +---------------------------- 3 files changed, 85 insertions(+), 59 deletions(-) diff --git a/app/models/legacy/dnskey_history.rb b/app/models/legacy/dnskey_history.rb index 1f62b1002..32a9cd855 100644 --- a/app/models/legacy/dnskey_history.rb +++ b/app/models/legacy/dnskey_history.rb @@ -4,5 +4,50 @@ module Legacy belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id + has_one :object_history, foreign_key: :historyid, primary_key: :historyid + + + def self.for_at(keysetid, time) + return [] unless keysetid + + sql = %Q{select distinct dh.id, dh.keysetid, dh.flags, dh.protocol, dh.alg, dh.key, + first_value(history.valid_from) OVER (PARTITION BY key ORDER BY history.valid_from ASC NULLS FIRST) valid_from, + first_value(history.valid_to) OVER (PARTITION BY key ORDER BY history.valid_to DESC NULLS FIRST) valid_to + FROM dnskey_history dh JOIN history ON dh.historyid=history.id + WHERE dh.keysetid IN (#{keysetid}) + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + ORDER BY dh.id;} + find_by_sql(sql) + end + + def new_object_hash(old_domain, new_domain) + new_object_mains(new_domain).merge( + creator_str: old_domain.object_registry.try(:registrar).try(:name), + updator_str: old_domain.object_history.try(:registrar).try(:name) || old_domain.object_registry.try(:registrar).try(:name), + legacy_domain_id: old_domain.id, + legacy_keyset_id: keysetid, + updated_at: (!object_registry.try(:object_history) || object_registry.try(:object_history).read_attribute(:update).nil?) ? (try(:crdate)||Time.zone.now) : object_registry.try(:object_history).read_attribute(:update) + ) + end + + def new_object_mains(new_domain) + @new_object_mains ||= { + domain_id: new_domain.id, + flags: flags, + protocol: protocol, + alg: alg, + public_key: key + } + end + + def historical_data(old_domain, new_domain, time_attr = :valid_from) + { + whodunnit: old_domain.history_domain.user.try(:id), + object: nil, + object_changes: new_object_hash(old_domain, new_domain).each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: [try(time_attr), old_domain.try(time_attr)].max + } + end end end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 5ea18947a..44dacad90 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -2,6 +2,7 @@ module Legacy class DomainHistory < Db self.table_name = :domain_history self.primary_key = :id + class_attribute :dnssecs belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id @@ -123,6 +124,44 @@ module Legacy end end + # returns imported dnskey ids + def import_dnskeys_history(new_domain, time) + self.class.dnssecs ||= {} + self.class.dnssecs[id] ||= {} + ids = [] + Legacy::DnskeyHistory.for_at(keyset, time).each do |dns| + # checking if we have create history for dnskey (cache) + if val = self.class.dnssecs[id][dns] + ids << val + else # if not found we should check current dnssec and historical if changes were done + # if current object wan't changed + if item=::Dnskey.where(dns.new_object_mains(new_domain)).first + item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + self.class.dnssecs[id][dns] = item.id + ids << item.id + # if current object was changed + elsif (versions = ::DnskeyVersion.where("object->>'legacy_domain_id'='#{id}'").to_a).any? + versions.each do |v| + if v.object.slice(*dns.new_object_mains(new_domain).stringify_keys.keys) == dns.new_object_mains(new_domain).keys + self.class.dnssecs[id][dns] = v.item_id + ids << v.item_id + v.item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + end + end + else + item=::Dnskey.new(id: ::Dnskey.next_id) + DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :destroy).first_or_create!(dns.historical_data(self, new_domain), :valid_to) if dns.valid_to + self.class.dnssecs[id][dns] = item.id + ids << item.id + end + end + end + + ids + end + + class << self def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index b982492cb..c9fad7768 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -2,7 +2,6 @@ namespace :import do desc 'Import all history' task history_all: :environment do Rake::Task['import:history_contacts'].invoke - Rake::Task['import:history_dnskeys'].invoke Rake::Task['import:history_domains'].invoke end @@ -78,63 +77,6 @@ namespace :import do end - desc 'Import contact history' - task history_dnskeys: :environment do - Legacy::DnskeyHistory.uniq.pluck(:id).each do |legacy_dnskey_id| - Dnskey.transaction do - contact = Dnskey.find_by(legacy_id: legacy_dnskey_id) - version_dns = DnskeytVersion.where("object->>'legacy_id' = '#{legacy_dnskey_id}'").select(:item_id).first - contact ||= Dnskey.new(id: version_dns.item_id, legacy_id: legacy_dnskey_id) if version_dns - contact ||= Dnskey.new(id: ::Contact.next_id, legacy_id: legacy_dnskey_id) - next if contact.versions.where(event: :create).any? - # add here to skip domains whith create history - - last_changes = nil - history = Legacy::ContactHistory.changes_dates_for(legacy_dnskey_id) - last_contact_action = history.sort.last[1].last # need to identify if we delete - - keys = history.keys.compact.sort - i = 0 - keys.each_with_index do |time| - history[time].each do |orig_history_klass| - changes = {} - responder = orig_history_klass[:klass].get_record_at(legacy_dnskey_id, orig_history_klass[:id]) - new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param]) - new_attrs[:id] = contact.id - - event = :update - event = :create if i == 0 - if orig_history_klass == last_contact_action && responder.valid_to.present? - event = :destroy - new_attrs = {} - end - - new_attrs.each do |k, v| - if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end - end - next if changes.blank? && event != :destroy - obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) - - hash = { - item_type: Dnskey.to_s, - item_id: contact.id, - event: event, - whodunnit: user.try(:id), - object: last_changes, - object_changes: changes, - created_at: time - } - DnskeyVersion.create!(hash) - - last_changes = new_attrs - i += 1 - end - end - end - end - end - desc 'Import domain history' task history_domains: :environment do @@ -200,7 +142,7 @@ namespace :import do admin_contacts: responder.history_domain.get_admin_contact_new_ids, tech_contacts: responder.history_domain.get_tech_contact_new_ids, nameservers: [], - dnskeys: [], + dnskeys: responder.history_domain.import_dnskeys_history(domain, time), registrant: [responder.history_domain.new_registrant_id] } ) From 478e07b5183d4047c3ae192ae85e2778b5e42ae1 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Tue, 15 Dec 2015 13:40:22 +0200 Subject: [PATCH 22/31] Story 108521790 importing dnskey (add comment for more understanding by others) --- app/models/legacy/domain_history.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 44dacad90..6852fd4bf 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -148,6 +148,7 @@ module Legacy v.item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) end end + # if no history was here else item=::Dnskey.new(id: ::Dnskey.next_id) DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) From faa69eee1c3043f3b1515fedce6e66c88f25db73 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 16 Dec 2015 11:46:52 +0200 Subject: [PATCH 23/31] Story 108521790 Update how nameservers history is handled --- app/models/legacy/dnskey_history.rb | 2 +- app/models/legacy/domain_history.rb | 155 +++++++++++++++++----------- lib/tasks/import_history.rake | 3 +- 3 files changed, 98 insertions(+), 62 deletions(-) diff --git a/app/models/legacy/dnskey_history.rb b/app/models/legacy/dnskey_history.rb index 32a9cd855..70f21a194 100644 --- a/app/models/legacy/dnskey_history.rb +++ b/app/models/legacy/dnskey_history.rb @@ -43,7 +43,7 @@ module Legacy def historical_data(old_domain, new_domain, time_attr = :valid_from) { - whodunnit: old_domain.history_domain.user.try(:id), + whodunnit: old_domain.user.try(:id), object: nil, object_changes: new_object_hash(old_domain, new_domain).each_with_object({}){|(k,v), h| h[k] = [nil, v]}, created_at: [try(time_attr), old_domain.try(time_attr)].max diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 6852fd4bf..8d84eeea0 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -3,6 +3,7 @@ module Legacy self.table_name = :domain_history self.primary_key = :id class_attribute :dnssecs + class_attribute :namesrvs belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id @@ -65,63 +66,104 @@ module Legacy # returns imported nameserver ids def import_nameservers_history(new_domain, time) - # #removing previous nameservers - # NameserverVersion.where("object_changes->>'legacy_domain_id' = '[nil,#{id}]'").where(event: :create).where("created_at <= ?", time).each do |nv| - # if NameserverVersion.where(item_type: nv.item_type, item_id: nv.item_id, event: :destroy).none? - # NameserverVersion.create!( - # item_type: nv.item_type, - # item_id: nv.item_id, - # event: :destroy, - # whodunnit: user.try(:id), - # object: nv.object_changes.each_with_object({}){|(k,v),hash| hash[k] = v.last }, - # object_changes: {}, - # created_at: time - # ) - # end - # end + self.class.namesrvs ||= {} + self.class.namesrvs[id] ||= {} + ids = [] - - if (nssets = nsset_histories.at(time).to_a).any? - ids = [] - nssets.each do |nsset| - nsset.host_histories.at(time).each do |host| - ips = {ipv4: [],ipv6: []} - host.host_ipaddr_map_histories.at(time).each do |ip_map| - next unless ip_map.ipaddr - ips[:ipv4] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv4? - ips[:ipv6] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv6? - end - - server = { - id: Nameserver.next_id, - hostname: host.fqdn.try(:strip), - ipv4: ips[:ipv4], - ipv6: ips[:ipv6], - creator_str: object_registry.try(:registrar).try(:name), - updator_str: object_history.try(:registrar).try(:name) || object_registry.try(:registrar).try(:name), - legacy_domain_id: id, - domain_id: new_domain.id, - created_at: nsset.object_registry.try(:crdate), - updated_at: nsset.object_registry.try(:object_history).read_attribute(:update) || nsset.object_registry.try(:crdate) - } - - NameserverVersion.create!( - item_type: Nameserver.to_s, - item_id: server[:id], - event: :create, - whodunnit: user.try(:id), - object: nil, - object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, - created_at: time - ) - ids << server[:id] + nsset_histories.at(time).to_a.each do |nsset| + nsset.host_histories.at(time).each do |host| + ips = {ipv4: [],ipv6: []} + host.host_ipaddr_map_histories.where.not(ipaddr: nil).at(time).each do |ip_map| + ips[:ipv4] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv4? + ips[:ipv6] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv6? end + main_attrs = { + hostname: SimpleIDN.to_unicode(host.fqdn.try(:strip)), + ipv4: ips[:ipv4].sort, + ipv6: ips[:ipv6].sort, + legacy_domain_id: id, + domain_id: new_domain.id, + } + server = main_attrs.merge( + creator_str: object_registry.try(:registrar).try(:name), + updator_str: object_history.try(:registrar).try(:name) || object_registry.try(:registrar).try(:name), + created_at: nsset.object_registry.try(:crdate), + updated_at: nsset.object_registry.try(:object_history).read_attribute(:update) || nsset.object_registry.try(:crdate) + ) + + + if val = self.class.namesrvs[id][main_attrs] + ids << val + else # if not found we should check current dnssec and historical if changes were done + # firstly we need to select the first historical object to take the earliest from create or destroy + if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").where("object->>'hostname'='#{main_attrs[:hostname]}'").reorder("created_at ASC").first + server[:id] = version.item_id.to_i + version.item.versions.where(event: :create).first_or_create!( + whodunnit: user.try(:id), + object: nil, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: time + ) + if !version.ipv4.sort.eql?(main_attrs[:ipv4]) || !version.ipv6.sort.eql?(main_attrs[:ipv6]) + object_changes = {} + server.stringify_keys.each{|k, v| object_changes[k] = [v, version.object[k]] if v != version.object[k] } + version.item.versions.where(event: :update).create!( + whodunnit: user.try(:id), + object: server, + object_changes: object_changes, + created_at: time + ) + end + + # if no historical data - try to load existing + elsif (list = ::Nameserver.where(domain_id: main_attrs[:domain_id], legacy_domain_id: main_attrs[:legacy_domain_id], hostname: main_attrs[:hostname]).to_a).any? + if new_no_version = list.detect{|e|e.versions.where(event: :create).none?} # no create version, so was created via import + server[:id] = new_no_version.id.to_i + new_no_version.versions.where(event: :create).first_or_create!( + whodunnit: user.try(:id), + object: nil, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: time + ) + if !new_no_version.ipv4.sort.eql?(main_attrs[:ipv4]) || !new_no_version.ipv6.sort.eql?(main_attrs[:ipv6]) + object_changes = {} + server.stringify_keys.each{|k, v| object_changes[k] = [v, new_no_version.attributes[k]] if v != new_no_version.attributes[k] } + new_no_version.versions.where(event: :update).create!( + whodunnit: user.try(:id), + object: server, + object_changes: object_changes, + created_at: time + ) + end + else + server[:id] = ::Nameserver.next_id + create_nameserver_history(server,time) + end + + else + server[:id] = ::Nameserver.next_id + create_nameserver_history(server,time) + end + self.class.namesrvs[id][main_attrs] = server[:id] + ids << server[:id] + + end end - ids - else - [] + end + ids + end + + def create_nameserver_history server, time + ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :create).first_or_create!( + whodunnit: user.try(:id), object: nil, created_at: time, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + ) + ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :destroy).create!( + whodunnit: user.try(:id), object: server, created_at: Time.now + 2.days, + object_changes: {}, + ) end # returns imported dnskey ids @@ -134,7 +176,7 @@ module Legacy if val = self.class.dnssecs[id][dns] ids << val else # if not found we should check current dnssec and historical if changes were done - # if current object wan't changed + # if current object wasn't changed if item=::Dnskey.where(dns.new_object_mains(new_domain)).first item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) self.class.dnssecs[id][dns] = item.id @@ -184,12 +226,7 @@ module Legacy end - # def last_history_action domain_id - # sql = %Q{SELECT dh.*, h.valid_from, h.valid_to - # from domain_history dh JOIN history h ON dh.historyid=h.id - # where dh.id=#{domain_id} order by dh.historyid desc limit 1;} - # find_by_sql(sql).first - # end + end end end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index c9fad7768..ce9480ef2 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -128,7 +128,6 @@ namespace :import do if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end end next if changes.blank? && event != :destroy - responder.import_nameservers_history(domain, time) if responder.respond_to?(:import_nameservers_history) DomainVersion.create!( item_type: domain.class, @@ -141,7 +140,7 @@ namespace :import do children: { admin_contacts: responder.history_domain.get_admin_contact_new_ids, tech_contacts: responder.history_domain.get_tech_contact_new_ids, - nameservers: [], + nameservers: responder.history_domain.import_nameservers_history(domain, time), dnskeys: responder.history_domain.import_dnskeys_history(domain, time), registrant: [responder.history_domain.new_registrant_id] } From b75b35f5546d1dbaba7972abd172a4abdc2125b9 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 18 Dec 2015 08:51:03 +0200 Subject: [PATCH 24/31] rm debugging info --- lib/tasks/import_history.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index ce9480ef2..aab50c177 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -81,7 +81,7 @@ namespace :import do desc 'Import domain history' task history_domains: :environment do Domain.transaction do - Legacy::DomainHistory.uniq.where(id: 294516).pluck(:id).each do |legacy_domain_id| + Legacy::DomainHistory.uniq.pluck(:id).each do |legacy_domain_id| domain = Domain.find_by(legacy_id: legacy_domain_id) version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain From 1fb0259806191c9ac9d34f94c248b9f54d369870 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 6 Jan 2016 15:18:30 +0200 Subject: [PATCH 25/31] Story#108602614 - optimize contacts history import --- app/models/legacy/contact_history.rb | 6 +++--- app/models/legacy/domain.rb | 5 +++++ app/models/legacy/domain_history.rb | 11 +++++++---- app/models/legacy/object_state.rb | 6 +++--- db/migrate/20160106101725_version_object_is_jsonb.rb | 10 ++++++++++ lib/tasks/import_history.rake | 11 +++++++---- 6 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 db/migrate/20160106101725_version_object_is_jsonb.rb diff --git a/app/models/legacy/contact_history.rb b/app/models/legacy/contact_history.rb index 040ba0dec..41f664d6e 100644 --- a/app/models/legacy/contact_history.rb +++ b/app/models/legacy/contact_history.rb @@ -28,7 +28,7 @@ module Legacy ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype], auth_info: x.object_history.authinfopw.try(:strip), name: name, - registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), creator_str: x.object_registry.try(:registrar).try(:name), updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), legacy_id: x.id, @@ -43,13 +43,13 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT dh.*, valid_from + sql = %Q{SELECT dh.historyid, valid_from, valid_to FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} - # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} hash = {} find_by_sql(sql).each do |rec| hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to end hash end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index 5725c3142..949a14702 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -25,5 +25,10 @@ module Legacy # OK status is default domain_statuses << DomainStatus::OK if domain_statuses.empty? end + + def self.new_registrar_cached old_id + @new_registrar_cache ||= {} + @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id) + end end end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 8d84eeea0..4b75b81da 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -18,7 +18,7 @@ module Legacy x = self { name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), - registrar_id: ::Registrar.find_by(legacy_id: x.object_history.try(:clid)).try(:id), + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), registrant_id: new_registrant_id, registered_at: x.object_registry.try(:crdate), valid_from: x.object_registry.try(:crdate), @@ -55,7 +55,7 @@ module Legacy def user @user ||= begin obj_his = Legacy::ObjectHistory.find_by(historyid: historyid) - Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) end end @@ -97,7 +97,10 @@ module Legacy ids << val else # if not found we should check current dnssec and historical if changes were done # firstly we need to select the first historical object to take the earliest from create or destroy - if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'").where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'").where("object->>'hostname'='#{main_attrs[:hostname]}'").reorder("created_at ASC").first + if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'"). + where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'"). + where("object->>'hostname'='#{main_attrs[:hostname]}'"). + reorder("created_at ASC").first server[:id] = version.item_id.to_i version.item.versions.where(event: :create).first_or_create!( whodunnit: user.try(:id), @@ -209,11 +212,11 @@ module Legacy def changes_dates_for domain_id sql = %Q{SELECT dh.*, valid_from FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} - # find_by_sql(sql).map{|e| e.attributes.values_at("valid_from") }.flatten.each_with_object({}){|e,h|h[e.try(:to_f)] = [self]} hash = {} find_by_sql(sql).each do |rec| hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to end hash end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index df892866d..31ea6b408 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -101,7 +101,7 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.*, + sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.valid_from, state.valid_to, extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from @@ -127,7 +127,7 @@ module Legacy end def states_for_domain_at(domain_id, time) - sql = %Q{SELECT state.* + sql = %Q{SELECT state.state_id FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) @@ -143,7 +143,7 @@ module Legacy def states_for_contact_at(contact_id, time) - sql = %Q{SELECT state.* + sql = %Q{SELECT state.state_id FROM object_history t_2 JOIN object_state state ON (t_2.historyid >= state.ohid_from AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) diff --git a/db/migrate/20160106101725_version_object_is_jsonb.rb b/db/migrate/20160106101725_version_object_is_jsonb.rb new file mode 100644 index 000000000..68212d2b6 --- /dev/null +++ b/db/migrate/20160106101725_version_object_is_jsonb.rb @@ -0,0 +1,10 @@ +class VersionObjectIsJsonb < ActiveRecord::Migration + def up + change_column :log_contacts, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) + add_index :registrars, :legacy_id + end + def down + change_column :log_contacts, :object, :json, using: "object::json" + end +end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index aab50c177..35ab7ff07 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -8,8 +8,10 @@ namespace :import do desc 'Import contact history' task history_contacts: :environment do - Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| + Legacy::ContactHistory.uniq.where("id > 4175").pluck(:id).each do |legacy_contact_id| + start = Time.now.to_f Contact.transaction do + data = [] contact = Contact.find_by(legacy_id: legacy_contact_id) version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact @@ -55,7 +57,7 @@ namespace :import do end next if changes.blank? && event != :destroy obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Registrar.find_by(legacy_id: obj_his.upid || obj_his.clid).try(:api_users).try(:first) + user = Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) hash = { item_type: Contact.to_s, @@ -66,13 +68,15 @@ namespace :import do object_changes: changes, created_at: time } - ContactVersion.create!(hash) + data << hash last_changes = new_attrs i += 1 end end + ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any? end + puts "Legacy Contact id #{legacy_contact_id} finished in #{Time.now.to_f - start}" end end @@ -115,7 +119,6 @@ namespace :import do new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) new_attrs[:id] = domain.id new_attrs[:updated_at] = time - p time event = :update event = :create if i == 0 From 795befb02573e661bbf38fe47be4b5cd89688c09 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 6 Jan 2016 15:18:58 +0200 Subject: [PATCH 26/31] Story#108602614 - optimize contacts history import --- lib/tasks/import_history.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 35ab7ff07..535ef55ec 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -8,7 +8,7 @@ namespace :import do desc 'Import contact history' task history_contacts: :environment do - Legacy::ContactHistory.uniq.where("id > 4175").pluck(:id).each do |legacy_contact_id| + Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| start = Time.now.to_f Contact.transaction do data = [] From f02ebe5559a82a95f2ac550c2f277bd6deed5398 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Wed, 6 Jan 2016 15:58:17 +0200 Subject: [PATCH 27/31] Story#108602614 - optimize logs history indexing --- .../20160106101725_version_object_is_jsonb.rb | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/db/migrate/20160106101725_version_object_is_jsonb.rb b/db/migrate/20160106101725_version_object_is_jsonb.rb index 68212d2b6..399a212b8 100644 --- a/db/migrate/20160106101725_version_object_is_jsonb.rb +++ b/db/migrate/20160106101725_version_object_is_jsonb.rb @@ -2,9 +2,25 @@ class VersionObjectIsJsonb < ActiveRecord::Migration def up change_column :log_contacts, :object, :jsonb, using: "object::jsonb" execute %q(CREATE INDEX "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) - add_index :registrars, :legacy_id + change_column :log_domains, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_domains_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) + + change_column :log_dnskeys, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_dnskeys_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int))) + change_column :log_nameservers, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_nameservers_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int))) + + add_index :registrars, :legacy_id rescue true end def down - change_column :log_contacts, :object, :json, using: "object::json" + change_column :log_contacts, :object, :json, using: "object::json" + change_column :log_domains, :object, :json, using: "object::json" + change_column :log_dnskeys, :object, :json, using: "object::json" + change_column :log_nameservers, :object, :json, using: "object::json" + + drop_index :log_contacts_object_legacy_id + drop_index :log_domains_object_legacy_id + drop_index :log_dnskeys_object_legacy_id + drop_index :log_nameservers_object_legacy_id end end From 34aa3c074122ee922d625771ab6b0d47f6e47fbd Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Thu, 7 Jan 2016 09:20:24 +0200 Subject: [PATCH 28/31] Story#108602614 - one more speed optimization --- app/models/legacy/domain.rb | 6 ++++++ app/models/legacy/domain_history.rb | 2 +- lib/tasks/import_history.rake | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index 949a14702..c17033c29 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -30,5 +30,11 @@ module Legacy @new_registrar_cache ||= {} @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id) end + + def self.new_api_user_cached old_id + @new_api_user_cache ||= {} + @new_api_user_cache[old_id] ||= Legacy::Domain.new_registrar_cached(old_id).try(:api_users).try(:first) + end + end end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 4b75b81da..bbe33aaa0 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -55,7 +55,7 @@ module Legacy def user @user ||= begin obj_his = Legacy::ObjectHistory.find_by(historyid: historyid) - Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) + Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid) end end diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 535ef55ec..0f41d0a5c 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -57,7 +57,7 @@ namespace :import do end next if changes.blank? && event != :destroy obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) - user = Legacy::Domain.new_registrar_cached(obj_his.upid || obj_his.clid).try(:api_users).try(:first) + user = Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid) hash = { item_type: Contact.to_s, From 1e88e27355a45ef98344ba0b9930632bcf44124c Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Fri, 15 Jan 2016 14:42:16 +0200 Subject: [PATCH 29/31] Story#108602614 - parallel history import --- Gemfile | 1 + Gemfile.lock | 2 ++ lib/tasks/import_history.rake | 37 +++++++++++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 10c680786..efa8b1f81 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ gem 'rails', '4.2.4' # when update, all initializers eis_custom files nee gem 'iso8601', '0.8.6' # for dates and times gem 'hashie-forbidden_attributes', '0.1.1' gem 'SyslogLogger', '2.0', require: 'syslog/logger' +gem 'parallel' # load env gem 'figaro', '1.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index b92d6dc1b..4b7c74561 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -330,6 +330,7 @@ GEM nprogress-rails (0.1.6.7) open4 (1.3.4) orm_adapter (0.5.0) + parallel (1.6.1) parser (2.2.2.6) ast (>= 1.1, < 3.0) pdfkit (0.6.2) @@ -603,6 +604,7 @@ DEPENDENCIES nokogiri (= 1.6.6.2) nprogress-rails (= 0.1.6.7) paper_trail! + parallel pdfkit (= 0.6.2) pg (= 0.18.2) phantomjs (= 1.9.8.0) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 0f41d0a5c..5b6e80230 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -5,10 +5,39 @@ namespace :import do Rake::Task['import:history_domains'].invoke end + def parallel_import all_ids + thread_pool = (Parallel.processor_count rescue 4) - 1 + threads = [] + + all_ids.each_with_index do |one_id, i| + process = Process.fork do + begin + yield(one_id, i) + rescue => e + Rails.logger.error("[EXCEPTION] #{Process.pid}") + Rails.logger.error("#{Process.pid} #{e.message}" ) + Rails.logger.error("#{Process.pid} #{e.backtrace.join("\n")}") + ensure + ActiveRecord::Base.remove_connection + Process.exit! + end + end + + threads << process + if threads.count >= thread_pool + threads.delete(Process.wait(0)) + end + end + + Process.waitall + end + desc 'Import contact history' task history_contacts: :environment do - Legacy::ContactHistory.uniq.pluck(:id).each do |legacy_contact_id| + old_ids = Legacy::ContactHistory.uniq.pluck(:id) + old_size = old_ids.size + parallel_import(old_ids) do |legacy_contact_id, process_idx| start = Time.now.to_f Contact.transaction do data = [] @@ -76,7 +105,7 @@ namespace :import do end ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any? end - puts "Legacy Contact id #{legacy_contact_id} finished in #{Time.now.to_f - start}" + puts "[PID: #{Process.pid}] Legacy Contact #{legacy_contact_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}" end end @@ -84,8 +113,8 @@ namespace :import do desc 'Import domain history' task history_domains: :environment do - Domain.transaction do - Legacy::DomainHistory.uniq.pluck(:id).each do |legacy_domain_id| + parallel_import(Legacy::DomainHistory.uniq.pluck(:id)) do |legacy_domain_id| + Domain.transaction do domain = Domain.find_by(legacy_id: legacy_domain_id) version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain From 326f4ad4ee0cc692750045598ad9dfeaaec22270 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Mon, 18 Jan 2016 11:17:14 +0200 Subject: [PATCH 30/31] 104525318 - default legacy legal docs conf --- config/application-example.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/application-example.yml b/config/application-example.yml index b08a1b007..a1f71d78c 100644 --- a/config/application-example.yml +++ b/config/application-example.yml @@ -37,6 +37,7 @@ app_name: '.EE Registry' zonefile_export_dir: 'export/zonefiles' bank_statement_import_dir: 'import/bank_statements' legal_documents_dir: 'import/legal_documents' +legacy_legal_documents_dir: 'import/legacy_legal_documents' time_zone: 'Tallinn' # more zones by rake time:zones:all openssl_config_path: '/etc/ssl/openssl.cnf' From 40d4ed4408639f720d72626ccfe685809bb88834 Mon Sep 17 00:00:00 2001 From: Vladimir Krylov Date: Mon, 18 Jan 2016 13:02:57 +0200 Subject: [PATCH 31/31] Story#108521790 - domain cahnges_dates_for has valid_to --- app/models/legacy/domain_history.rb | 2 +- lib/tasks/import_history.rake | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index bbe33aaa0..e052c4616 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -210,7 +210,7 @@ module Legacy class << self def changes_dates_for domain_id - sql = %Q{SELECT dh.*, valid_from + sql = %Q{SELECT dh.*, valid_from, valid_to FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} hash = {} diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake index 5b6e80230..51cc04ed2 100644 --- a/lib/tasks/import_history.rake +++ b/lib/tasks/import_history.rake @@ -113,7 +113,10 @@ namespace :import do desc 'Import domain history' task history_domains: :environment do - parallel_import(Legacy::DomainHistory.uniq.pluck(:id)) do |legacy_domain_id| + old_ids = Legacy::DomainHistory.uniq.pluck(:id) + old_size = old_ids.size + parallel_import(old_ids) do |legacy_domain_id, process_idx| + start = Time.now.to_f Domain.transaction do domain = Domain.find_by(legacy_id: legacy_domain_id) version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first @@ -183,8 +186,8 @@ namespace :import do end end end + puts "[PID: #{Process.pid}] Legacy Domain #{legacy_domain_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}" end - end