diff --git a/Gemfile b/Gemfile index 10c680786..efa8b1f81 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ gem 'rails', '4.2.4' # when update, all initializers eis_custom files nee gem 'iso8601', '0.8.6' # for dates and times gem 'hashie-forbidden_attributes', '0.1.1' gem 'SyslogLogger', '2.0', require: 'syslog/logger' +gem 'parallel' # load env gem 'figaro', '1.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index b92d6dc1b..4b7c74561 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -330,6 +330,7 @@ GEM nprogress-rails (0.1.6.7) open4 (1.3.4) orm_adapter (0.5.0) + parallel (1.6.1) parser (2.2.2.6) ast (>= 1.1, < 3.0) pdfkit (0.6.2) @@ -603,6 +604,7 @@ DEPENDENCIES nokogiri (= 1.6.6.2) nprogress-rails (= 0.1.6.7) paper_trail! + parallel pdfkit (= 0.6.2) pg (= 0.18.2) phantomjs (= 1.9.8.0) diff --git a/app/models/contact.rb b/app/models/contact.rb index ccc44851d..98e273cf9 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -225,6 +225,10 @@ class Contact < ActiveRecord::Base kit.to_pdf end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end def roid diff --git a/app/models/dnskey.rb b/app/models/dnskey.rb index 02b43d729..ef8f68a39 100644 --- a/app/models/dnskey.rb +++ b/app/models/dnskey.rb @@ -120,5 +120,9 @@ class Dnskey < ActiveRecord::Base def bin_to_hex(s) s.each_byte.map { |b| format('%02X', b) }.join end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end end diff --git a/app/models/domain.rb b/app/models/domain.rb index e4882473a..728b5e688 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -221,6 +221,10 @@ class Domain < ActiveRecord::Base ) end + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end + # rubocop: disable Metrics/AbcSize # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity @@ -821,8 +825,8 @@ class Domain < ActiveRecord::Base log[:admin_contacts] = admin_contact_ids log[:tech_contacts] = tech_contact_ids log[:nameservers] = nameserver_ids + log[:dnskeys] = dnskey_ids log[:registrant] = [registrant_id] - log[:domain_statuses] = domain_status_ids log end diff --git a/app/models/epp/domain.rb b/app/models/epp/domain.rb index 15e16163a..b64f57683 100644 --- a/app/models/epp/domain.rb +++ b/app/models/epp/domain.rb @@ -698,6 +698,9 @@ class Epp::Domain < Domain end attach_legal_document(self.class.parse_legal_document_from_frame(frame)) + + # we want to transfer data to new owner at any case. + # We also hope that if domain is not valid, new registrar would be better. save!(validate: false) return dt diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 04f8c34a4..fca3a2b23 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -1,5 +1,12 @@ module Legacy class Contact < Db + IDENT_TYPE_MAP = { + 2 => ::Contact::PRIV, + 3 => ::Contact::PASSPORT, + 4 => ::Contact::ORG, + 6 => ::Contact::BIRTHDAY + } + self.table_name = :contact belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id diff --git a/app/models/legacy/contact_history.rb b/app/models/legacy/contact_history.rb new file mode 100644 index 000000000..41f664d6e --- /dev/null +++ b/app/models/legacy/contact_history.rb @@ -0,0 +1,66 @@ +module Legacy + class ContactHistory < Db + self.table_name = :contact_history + self.primary_key = :id + + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id + belongs_to :contact, foreign_key: :id + belongs_to :history, foreign_key: :historyid + has_one :object_history, foreign_key: :historyid, primary_key: :historyid + + def get_current_contact_object(time, change_param) + x = self + if 4 == x.ssntype + name = x.organization.try(:strip).presence || x.name.try(:strip).presence + else + name = x.name.try(:strip).presence || x.organization.try(:strip).presence + end + + { + code: x.object_registry.name.try(:strip), + phone: x.telephone.try(:strip), + email: [x.email.try(:strip), x.notifyemail.try(:strip)].uniq.select(&:present?).join(', '), + fax: x.fax.try(:strip), + created_at: x.object_registry.try(:crdate), + updated_at: x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update), + ident: x.ssn.try(:strip), + ident_type: ::Legacy::Contact::IDENT_TYPE_MAP[x.ssntype], + auth_info: x.object_history.authinfopw.try(:strip), + name: name, + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), + creator_str: x.object_registry.try(:registrar).try(:name), + updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), + legacy_id: x.id, + street: [x.street1.try(:strip), x.street2.try(:strip), x.street3.try(:strip)].compact.join(", "), + city: x.city.try(:strip), + zip: x.postalcode.try(:strip), + state: x.stateorprovince.try(:strip), + country_code: x.country.try(:strip), + statuses: ::Legacy::ObjectState.states_for_contact_at(x.id, time) + } + end + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT dh.historyid, valid_from, valid_to + FROM contact_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} + + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to + end + hash + end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT dh.*, h.valid_from, h.valid_to + from contact_history dh JOIN history h ON dh.historyid=h.id + where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} + find_by_sql(sql).first + end + + end + end +end diff --git a/app/models/legacy/dnskey.rb b/app/models/legacy/dnskey.rb index 3fa47f48b..782673ef8 100644 --- a/app/models/legacy/dnskey.rb +++ b/app/models/legacy/dnskey.rb @@ -1,8 +1,10 @@ module Legacy class Dnskey < Db self.table_name = :dnskey + self.primary_key = :id belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id + has_one :object_history, foreign_key: :historyid, primary_key: :historyid end end diff --git a/app/models/legacy/dnskey_history.rb b/app/models/legacy/dnskey_history.rb new file mode 100644 index 000000000..70f21a194 --- /dev/null +++ b/app/models/legacy/dnskey_history.rb @@ -0,0 +1,53 @@ +module Legacy + class DnskeyHistory < Db + self.table_name = :dnskey_history + + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id + has_one :object_history, foreign_key: :historyid, primary_key: :historyid + + + def self.for_at(keysetid, time) + return [] unless keysetid + + sql = %Q{select distinct dh.id, dh.keysetid, dh.flags, dh.protocol, dh.alg, dh.key, + first_value(history.valid_from) OVER (PARTITION BY key ORDER BY history.valid_from ASC NULLS FIRST) valid_from, + first_value(history.valid_to) OVER (PARTITION BY key ORDER BY history.valid_to DESC NULLS FIRST) valid_to + FROM dnskey_history dh JOIN history ON dh.historyid=history.id + WHERE dh.keysetid IN (#{keysetid}) + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + ORDER BY dh.id;} + find_by_sql(sql) + end + + def new_object_hash(old_domain, new_domain) + new_object_mains(new_domain).merge( + creator_str: old_domain.object_registry.try(:registrar).try(:name), + updator_str: old_domain.object_history.try(:registrar).try(:name) || old_domain.object_registry.try(:registrar).try(:name), + legacy_domain_id: old_domain.id, + legacy_keyset_id: keysetid, + updated_at: (!object_registry.try(:object_history) || object_registry.try(:object_history).read_attribute(:update).nil?) ? (try(:crdate)||Time.zone.now) : object_registry.try(:object_history).read_attribute(:update) + ) + end + + def new_object_mains(new_domain) + @new_object_mains ||= { + domain_id: new_domain.id, + flags: flags, + protocol: protocol, + alg: alg, + public_key: key + } + end + + def historical_data(old_domain, new_domain, time_attr = :valid_from) + { + whodunnit: old_domain.user.try(:id), + object: nil, + object_changes: new_object_hash(old_domain, new_domain).each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: [try(time_attr), old_domain.try(time_attr)].max + } + end + end +end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index d0a05d178..c17033c29 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -7,10 +7,34 @@ module Legacy belongs_to :nsset, foreign_key: :nsset # belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' - has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id + has_many :object_states, foreign_key: :object_id has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset has_many :domain_contact_maps, foreign_key: :domainid has_many :nsset_contact_maps, foreign_key: :nssetid, primary_key: :nsset has_many :domain_histories, foreign_key: :id + alias_method :history, :domain_histories + + + def new_states + domain_statuses = [] + object_states.valid.each do |state| + next if state.name.blank? + domain_statuses << state.name + end + + # OK status is default + domain_statuses << DomainStatus::OK if domain_statuses.empty? + end + + def self.new_registrar_cached old_id + @new_registrar_cache ||= {} + @new_registrar_cache[old_id] ||= ::Registrar.select(:id).find_by(legacy_id: old_id) + end + + def self.new_api_user_cached old_id + @new_api_user_cache ||= {} + @new_api_user_cache[old_id] ||= Legacy::Domain.new_registrar_cached(old_id).try(:api_users).try(:first) + end + end end diff --git a/app/models/legacy/domain_contact_map_history.rb b/app/models/legacy/domain_contact_map_history.rb new file mode 100644 index 000000000..9ad3a6f7e --- /dev/null +++ b/app/models/legacy/domain_contact_map_history.rb @@ -0,0 +1,5 @@ +module Legacy + class DomainContactMapHistory < Db + self.table_name = :domain_contact_map_history + end +end diff --git a/app/models/legacy/domain_history.rb b/app/models/legacy/domain_history.rb index 234585ac3..e052c4616 100644 --- a/app/models/legacy/domain_history.rb +++ b/app/models/legacy/domain_history.rb @@ -1,7 +1,235 @@ module Legacy class DomainHistory < Db self.table_name = :domain_history + self.primary_key = :id + class_attribute :dnssecs + class_attribute :namesrvs + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id belongs_to :domain, foreign_key: :id + belongs_to :history, foreign_key: :historyid + has_one :object_history, foreign_key: :historyid, primary_key: :historyid + has_many :nsset_histories, foreign_key: :id, primary_key: :nsset + has_many :domain_contact_map_histories, foreign_key: :historyid, primary_key: :historyid + has_many :nsset_contact_map_histories, foreign_key: :historyid, primary_key: :historyid + + def get_current_domain_object(time, change_param) + x = self + { + name: SimpleIDN.to_unicode(x.object_registry.name.try(:strip)), + registrar_id: ::Legacy::Domain.new_registrar_cached(x.object_history.try(:clid)).try(:id), + registrant_id: new_registrant_id, + registered_at: x.object_registry.try(:crdate), + valid_from: x.object_registry.try(:crdate), + valid_to: x.exdate, + auth_info: x.object_history.authinfopw.try(:strip), + created_at: x.object_registry.try(:crdate), + updated_at: x.object_history.read_attribute(:update).nil? ? x.object_registry.try(:crdate) : x.object_history.read_attribute(:update), + name_dirty: x.object_registry.name.try(:strip), + name_puny: SimpleIDN.to_ascii(x.object_registry.name.try(:strip)), + period: 1, + period_unit: 'y', + creator_str: x.object_registry.try(:registrar).try(:name), + updator_str: x.object_history.try(:registrar).try(:name) ? x.object_history.try(:registrar).try(:name) : x.object_registry.try(:registrar).try(:name), + legacy_id: x.id, + legacy_registrar_id: x.object_history.try(:clid), + legacy_registrant_id: x.registrant, + statuses: Legacy::ObjectState.states_for_domain_at(x.id, time) + } + end + + def get_admin_contact_new_ids + c_ids = domain_contact_map_histories.pluck(:contactid).join("','") + DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id) + end + def get_tech_contact_new_ids + c_ids = nsset_contact_map_histories.pluck(:contactid).join("','") + DomainVersion.where("object->>'legacy_id' IN ('#{c_ids}')").uniq.pluck(:item_id) + end + + def new_registrant_id + @new_registrant_id ||= ::Contact.find_by(legacy_id: registrant).try(:id) + end + + def user + @user ||= begin + obj_his = Legacy::ObjectHistory.find_by(historyid: historyid) + Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid) + end + end + + def history_domain + self + end + + + # returns imported nameserver ids + def import_nameservers_history(new_domain, time) + self.class.namesrvs ||= {} + self.class.namesrvs[id] ||= {} + ids = [] + + nsset_histories.at(time).to_a.each do |nsset| + nsset.host_histories.at(time).each do |host| + ips = {ipv4: [],ipv6: []} + host.host_ipaddr_map_histories.where.not(ipaddr: nil).at(time).each do |ip_map| + ips[:ipv4] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv4? + ips[:ipv6] << ip_map.ipaddr.to_s.strip if ip_map.ipaddr.ipv6? + end + + main_attrs = { + hostname: SimpleIDN.to_unicode(host.fqdn.try(:strip)), + ipv4: ips[:ipv4].sort, + ipv6: ips[:ipv6].sort, + legacy_domain_id: id, + domain_id: new_domain.id, + } + server = main_attrs.merge( + creator_str: object_registry.try(:registrar).try(:name), + updator_str: object_history.try(:registrar).try(:name) || object_registry.try(:registrar).try(:name), + created_at: nsset.object_registry.try(:crdate), + updated_at: nsset.object_registry.try(:object_history).read_attribute(:update) || nsset.object_registry.try(:crdate) + ) + + + if val = self.class.namesrvs[id][main_attrs] + ids << val + else # if not found we should check current dnssec and historical if changes were done + # firstly we need to select the first historical object to take the earliest from create or destroy + if version = ::NameserverVersion.where("object->>'domain_id'='#{main_attrs[:domain_id]}'"). + where("object->>'legacy_domain_id'='#{main_attrs[:legacy_domain_id]}'"). + where("object->>'hostname'='#{main_attrs[:hostname]}'"). + reorder("created_at ASC").first + server[:id] = version.item_id.to_i + version.item.versions.where(event: :create).first_or_create!( + whodunnit: user.try(:id), + object: nil, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: time + ) + if !version.ipv4.sort.eql?(main_attrs[:ipv4]) || !version.ipv6.sort.eql?(main_attrs[:ipv6]) + object_changes = {} + server.stringify_keys.each{|k, v| object_changes[k] = [v, version.object[k]] if v != version.object[k] } + version.item.versions.where(event: :update).create!( + whodunnit: user.try(:id), + object: server, + object_changes: object_changes, + created_at: time + ) + end + + # if no historical data - try to load existing + elsif (list = ::Nameserver.where(domain_id: main_attrs[:domain_id], legacy_domain_id: main_attrs[:legacy_domain_id], hostname: main_attrs[:hostname]).to_a).any? + if new_no_version = list.detect{|e|e.versions.where(event: :create).none?} # no create version, so was created via import + server[:id] = new_no_version.id.to_i + new_no_version.versions.where(event: :create).first_or_create!( + whodunnit: user.try(:id), + object: nil, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + created_at: time + ) + if !new_no_version.ipv4.sort.eql?(main_attrs[:ipv4]) || !new_no_version.ipv6.sort.eql?(main_attrs[:ipv6]) + object_changes = {} + server.stringify_keys.each{|k, v| object_changes[k] = [v, new_no_version.attributes[k]] if v != new_no_version.attributes[k] } + new_no_version.versions.where(event: :update).create!( + whodunnit: user.try(:id), + object: server, + object_changes: object_changes, + created_at: time + ) + end + else + server[:id] = ::Nameserver.next_id + create_nameserver_history(server,time) + end + + else + server[:id] = ::Nameserver.next_id + create_nameserver_history(server,time) + end + self.class.namesrvs[id][main_attrs] = server[:id] + ids << server[:id] + + end + end + + end + ids + end + + def create_nameserver_history server, time + ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :create).first_or_create!( + whodunnit: user.try(:id), object: nil, created_at: time, + object_changes: server.each_with_object({}){|(k,v), h| h[k] = [nil, v]}, + ) + ::NameserverVersion.where(item_id: server[:id], item_type: ::Nameserver.to_s).where(event: :destroy).create!( + whodunnit: user.try(:id), object: server, created_at: Time.now + 2.days, + object_changes: {}, + ) + end + + # returns imported dnskey ids + def import_dnskeys_history(new_domain, time) + self.class.dnssecs ||= {} + self.class.dnssecs[id] ||= {} + ids = [] + Legacy::DnskeyHistory.for_at(keyset, time).each do |dns| + # checking if we have create history for dnskey (cache) + if val = self.class.dnssecs[id][dns] + ids << val + else # if not found we should check current dnssec and historical if changes were done + # if current object wasn't changed + if item=::Dnskey.where(dns.new_object_mains(new_domain)).first + item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + self.class.dnssecs[id][dns] = item.id + ids << item.id + # if current object was changed + elsif (versions = ::DnskeyVersion.where("object->>'legacy_domain_id'='#{id}'").to_a).any? + versions.each do |v| + if v.object.slice(*dns.new_object_mains(new_domain).stringify_keys.keys) == dns.new_object_mains(new_domain).keys + self.class.dnssecs[id][dns] = v.item_id + ids << v.item_id + v.item.versions.where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + end + end + # if no history was here + else + item=::Dnskey.new(id: ::Dnskey.next_id) + DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :create).first_or_create!(dns.historical_data(self, new_domain)) + DnskeyVersion.where(item_type: ::Dnskey.to_s, item_id: item.id).where(event: :destroy).first_or_create!(dns.historical_data(self, new_domain), :valid_to) if dns.valid_to + self.class.dnssecs[id][dns] = item.id + ids << item.id + end + end + end + + ids + end + + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT dh.*, valid_from, valid_to + FROM domain_history dh JOIN history h ON dh.historyid=h.id where dh.id=#{domain_id};} + + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.historyid, klass: self, param: :valid_to}] if rec.valid_to + end + hash + end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT dh.*, h.valid_from, h.valid_to + from domain_history dh JOIN history h ON dh.historyid=h.id + where dh.id=#{domain_id} and dh.historyid = #{rec_id} ;} + find_by_sql(sql).first + end + + + + end end end diff --git a/app/models/legacy/history.rb b/app/models/legacy/history.rb new file mode 100644 index 000000000..98ef464b0 --- /dev/null +++ b/app/models/legacy/history.rb @@ -0,0 +1,5 @@ +module Legacy + class History < Db + self.table_name = :history + end +end \ No newline at end of file diff --git a/app/models/legacy/host_history.rb b/app/models/legacy/host_history.rb new file mode 100644 index 000000000..437e476af --- /dev/null +++ b/app/models/legacy/host_history.rb @@ -0,0 +1,15 @@ +module Legacy + class HostHistory < Db + self.table_name = :host_history + self.primary_key = :id + + belongs_to :history, foreign_key: :historyid + has_many :host_ipaddr_maps, foreign_key: :hostid + has_many :host_ipaddr_map_histories, foreign_key: :hostid, primary_key: :id + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/legacy/host_ipaddr_map_history.rb b/app/models/legacy/host_ipaddr_map_history.rb new file mode 100644 index 000000000..9f84121f7 --- /dev/null +++ b/app/models/legacy/host_ipaddr_map_history.rb @@ -0,0 +1,12 @@ +module Legacy + class HostIpaddrMapHistory < Db + self.table_name = :host_ipaddr_map_history + self.primary_key = :id + belongs_to :history, foreign_key: :historyid + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/legacy/nsset_contact_map_history.rb b/app/models/legacy/nsset_contact_map_history.rb new file mode 100644 index 000000000..ffcb74793 --- /dev/null +++ b/app/models/legacy/nsset_contact_map_history.rb @@ -0,0 +1,5 @@ +module Legacy + class NssetContactMapHistory < Db + self.table_name = :nsset_contact_map_history + end +end diff --git a/app/models/legacy/nsset_history.rb b/app/models/legacy/nsset_history.rb new file mode 100644 index 000000000..1a88ebc12 --- /dev/null +++ b/app/models/legacy/nsset_history.rb @@ -0,0 +1,17 @@ +module Legacy + class NssetHistory < Db + self.table_name = :nsset_history + self.primary_key = :id + + belongs_to :object, foreign_key: :id + belongs_to :object_registry, foreign_key: :id + belongs_to :history, foreign_key: :historyid, primary_key: :id + has_many :hosts, foreign_key: :nssetid + has_many :host_histories, foreign_key: :nssetid, primary_key: :id + + def self.at(time) + joins(:history).where("(valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ)") + end + end +end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 379d4d175..31ea6b408 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -1,6 +1,9 @@ module Legacy class ObjectState < Db self.table_name = :object_state + attr_accessor :history_domain + + scope :valid, -> { where('valid_to IS NULL') } # legacy values. Just for log # 2 => "serverRenewProhibited", @@ -77,5 +80,81 @@ module Legacy map[state_id] end + + def get_current_domain_object(time, param) + d_his = Legacy::DomainHistory.get_record_at(object_id, historyid) + @history_domain = d_his + + hash = d_his.get_current_domain_object(time, param) + hash[:statuses] = Legacy::ObjectState.states_for_domain_at(object_id, time + 1) + + hash + end + + def get_current_contact_object(time, param) + d_his = Legacy::ContactHistory.get_record_at(object_id, historyid) + hash = d_his.get_current_contact_object(time, param) + hash[:statuses] = Legacy::ObjectState.states_for_contact_at(object_id, time + 1) + + hash + end + + class << self + def changes_dates_for domain_id + sql = %Q{SELECT distinct t_2.id, state.id state_dot_id, state.valid_from, state.valid_to, + extract(epoch from valid_from) valid_from_unix, extract(epoch from valid_to) valid_to_unix + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id};} + hash = {} + find_by_sql(sql).each do |rec| + hash[rec.valid_from.try(:to_time)] = [{id: rec.state_dot_id, klass: self, param: :valid_from}] if rec.valid_from + hash[rec.valid_to.try(:to_time)] = [{id: rec.state_dot_id, klass: self, param: :valid_to}] if rec.valid_to + end + hash + end + + def get_record_at domain_id, rec_id + sql = %Q{SELECT distinct t_2.historyid, state.* + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id} AND state.id = #{rec_id};} + find_by_sql(sql).first + end + + def states_for_domain_at(domain_id, time) + sql = %Q{SELECT state.state_id + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{domain_id} + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + } + arr = find_by_sql(sql).uniq + arr.map!(&:name) if arr.any? + arr.present? ? arr : [::DomainStatus::OK] + end + + + def states_for_contact_at(contact_id, time) + sql = %Q{SELECT state.state_id + FROM object_history t_2 + JOIN object_state state ON (t_2.historyid >= state.ohid_from + AND (t_2.historyid <= state.ohid_to OR state.ohid_to IS NULL)) + AND t_2.id = state.object_id + WHERE state.object_id=#{contact_id} + AND (valid_from is null or valid_from <= '#{time.to_s}'::TIMESTAMPTZ) + AND (valid_to is null or valid_to >= '#{time}'::TIMESTAMPTZ) + } + + (find_by_sql(sql).uniq.to_a.map(&:name) + [::Contact::OK]).compact.uniq + end + end end end diff --git a/app/models/nameserver.rb b/app/models/nameserver.rb index ccedf0880..245dc602e 100644 --- a/app/models/nameserver.rb +++ b/app/models/nameserver.rb @@ -98,5 +98,9 @@ class Nameserver < ActiveRecord::Base # ignoring ips rel end + + def next_id + self.connection.select_value("SELECT nextval('#{self.sequence_name}')") + end end end diff --git a/config/application-example.yml b/config/application-example.yml index b08a1b007..a1f71d78c 100644 --- a/config/application-example.yml +++ b/config/application-example.yml @@ -37,6 +37,7 @@ app_name: '.EE Registry' zonefile_export_dir: 'export/zonefiles' bank_statement_import_dir: 'import/bank_statements' legal_documents_dir: 'import/legal_documents' +legacy_legal_documents_dir: 'import/legacy_legal_documents' time_zone: 'Tallinn' # more zones by rake time:zones:all openssl_config_path: '/etc/ssl/openssl.cnf' diff --git a/db/migrate/20160106101725_version_object_is_jsonb.rb b/db/migrate/20160106101725_version_object_is_jsonb.rb new file mode 100644 index 000000000..399a212b8 --- /dev/null +++ b/db/migrate/20160106101725_version_object_is_jsonb.rb @@ -0,0 +1,26 @@ +class VersionObjectIsJsonb < ActiveRecord::Migration + def up + change_column :log_contacts, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_contacts_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) + change_column :log_domains, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_domains_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_id' as int))) + + change_column :log_dnskeys, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_dnskeys_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int))) + change_column :log_nameservers, :object, :jsonb, using: "object::jsonb" + execute %q(CREATE INDEX "log_nameservers_object_legacy_id" ON "log_contacts"(cast("object"->>'legacy_domain_id' as int))) + + add_index :registrars, :legacy_id rescue true + end + def down + change_column :log_contacts, :object, :json, using: "object::json" + change_column :log_domains, :object, :json, using: "object::json" + change_column :log_dnskeys, :object, :json, using: "object::json" + change_column :log_nameservers, :object, :json, using: "object::json" + + drop_index :log_contacts_object_legacy_id + drop_index :log_domains_object_legacy_id + drop_index :log_dnskeys_object_legacy_id + drop_index :log_nameservers_object_legacy_id + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 28786d278..1926237a7 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -364,16 +364,6 @@ namespace :import do legacy_contact_id ) - # rubocop: disable Lint/UselessAssignment - domain_status_columns = %w( - description - value - creator_str - updator_str - legacy_domain_id - ) - # rubocop: enable Lint/UselessAssignment - nameserver_columns = %w( hostname ipv4 @@ -398,7 +388,6 @@ namespace :import do domains, nameservers, dnskeys, domain_contacts = [], [], [], [] existing_domain_ids = Domain.pluck(:legacy_id) - user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 Legacy::Domain.includes( @@ -414,16 +403,6 @@ namespace :import do count += 1 begin - # domain statuses - domain_statuses = [] - x.object_states.each do |state| - next if state.name.blank? - domain_statuses << state.name - end - - # OK status is default - domain_statuses << DomainStatus::OK if domain_statuses.empty? - domains << [ x.object_registry.name.try(:strip), Registrar.find_by(legacy_id: x.object.try(:clid)).try(:id), @@ -442,7 +421,7 @@ namespace :import do x.id, x.object_registry.try(:crid), x.registrant, - domain_statuses + x.new_states ] # admin contacts @@ -774,6 +753,7 @@ namespace :import do puts "-----> Imported zones in #{(Time.zone.now.to_f - start).round(2)} seconds" end + end def parse_zone_ns_data(domain, zone) diff --git a/lib/tasks/import_history.rake b/lib/tasks/import_history.rake new file mode 100644 index 000000000..51cc04ed2 --- /dev/null +++ b/lib/tasks/import_history.rake @@ -0,0 +1,195 @@ +namespace :import do + desc 'Import all history' + task history_all: :environment do + Rake::Task['import:history_contacts'].invoke + Rake::Task['import:history_domains'].invoke + end + + def parallel_import all_ids + thread_pool = (Parallel.processor_count rescue 4) - 1 + threads = [] + + all_ids.each_with_index do |one_id, i| + process = Process.fork do + begin + yield(one_id, i) + rescue => e + Rails.logger.error("[EXCEPTION] #{Process.pid}") + Rails.logger.error("#{Process.pid} #{e.message}" ) + Rails.logger.error("#{Process.pid} #{e.backtrace.join("\n")}") + ensure + ActiveRecord::Base.remove_connection + Process.exit! + end + end + + threads << process + if threads.count >= thread_pool + threads.delete(Process.wait(0)) + end + end + + Process.waitall + end + + + desc 'Import contact history' + task history_contacts: :environment do + old_ids = Legacy::ContactHistory.uniq.pluck(:id) + old_size = old_ids.size + parallel_import(old_ids) do |legacy_contact_id, process_idx| + start = Time.now.to_f + Contact.transaction do + data = [] + contact = Contact.find_by(legacy_id: legacy_contact_id) + version_contact = ContactVersion.where("object->>'legacy_id' = '#{legacy_contact_id}'").select(:item_id).first + contact ||= Contact.new(id: version_contact.item_id, legacy_id: legacy_contact_id) if version_contact + contact ||= Contact.new(id: ::Contact.next_id, legacy_id: legacy_contact_id) + next if contact.versions.where(event: :create).any? + # add here to skip domains whith create history + + # 1. add domain changes + # 2. add states + # compose hash of change time -> Object changes + last_changes = nil + history = Legacy::ObjectState.changes_dates_for(legacy_contact_id) + con_his = Legacy::ContactHistory.changes_dates_for(legacy_contact_id) + last_contact_action = con_his.sort.last[1].last # need to identify if we delete + + # merging changes together + con_his.each do |time, klasses| + if history.has_key?(time) + history[time] = history[time] | klasses + else + history[time] = klasses + end + end + + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| + history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_contact_id, orig_history_klass[:id]) + new_attrs = responder.get_current_contact_object(time, orig_history_klass[:param]) + new_attrs[:id] = contact.id + + event = :update + event = :create if i == 0 + if orig_history_klass == last_contact_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? && event != :destroy + obj_his = Legacy::ObjectHistory.find_by(historyid: responder.historyid) + user = Legacy::Domain.new_api_user_cached(obj_his.upid || obj_his.clid) + + hash = { + item_type: Contact.to_s, + item_id: contact.id, + event: event, + whodunnit: user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time + } + data << hash + + last_changes = new_attrs + i += 1 + end + end + ContactVersion.import_without_validations_or_callbacks data.first.keys, data.map(&:values) if data.any? + end + puts "[PID: #{Process.pid}] Legacy Contact #{legacy_contact_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}" + end + end + + + + desc 'Import domain history' + task history_domains: :environment do + old_ids = Legacy::DomainHistory.uniq.pluck(:id) + old_size = old_ids.size + parallel_import(old_ids) do |legacy_domain_id, process_idx| + start = Time.now.to_f + Domain.transaction do + domain = Domain.find_by(legacy_id: legacy_domain_id) + version_domain = DomainVersion.where("object->>'legacy_id' = '#{legacy_domain_id}'").select(:item_id).first + domain ||= Domain.new(id: version_domain.item_id, legacy_id: legacy_domain_id) if version_domain + domain ||= Domain.new(id: ::Domain.next_id, legacy_id: legacy_domain_id) + next if domain.versions.where(event: :create).any? + # add here to skip domains whith create history + + # 1. add domain changes + # 2. add states + # compose hash of change time -> Object changes + last_changes = nil + history = Legacy::ObjectState.changes_dates_for(legacy_domain_id) + dom_his = Legacy::DomainHistory.changes_dates_for(legacy_domain_id) + last_domain_action = dom_his.sort.last[1].last # need to identify if we delete + + # merging changes together + dom_his.each do |time, klasses| + if history.has_key?(time) + history[time] = history[time] | klasses + else + history[time] = klasses + end + end + + keys = history.keys.compact.sort + i = 0 + keys.each_with_index do |time| + history[time].each do |orig_history_klass| + changes = {} + responder = orig_history_klass[:klass].get_record_at(legacy_domain_id, orig_history_klass[:id]) + new_attrs = responder.get_current_domain_object(time, orig_history_klass[:param]) + new_attrs[:id] = domain.id + new_attrs[:updated_at] = time + + event = :update + event = :create if i == 0 + if orig_history_klass == last_domain_action && responder.valid_to.present? + event = :destroy + new_attrs = {} + end + + new_attrs.each do |k, v| + if (old_val = last_changes.to_h[k]) != v then changes[k] = [old_val, v] end + end + next if changes.blank? && event != :destroy + + DomainVersion.create!( + item_type: domain.class, + item_id: domain.id, + event: event, + whodunnit: responder.history_domain.user.try(:id), + object: last_changes, + object_changes: changes, + created_at: time, + children: { + admin_contacts: responder.history_domain.get_admin_contact_new_ids, + tech_contacts: responder.history_domain.get_tech_contact_new_ids, + nameservers: responder.history_domain.import_nameservers_history(domain, time), + dnskeys: responder.history_domain.import_dnskeys_history(domain, time), + registrant: [responder.history_domain.new_registrant_id] + } + ) + + last_changes = new_attrs + i += 1 + end + end + end + puts "[PID: #{Process.pid}] Legacy Domain #{legacy_domain_id} (#{process_idx}/#{old_size}) finished in #{Time.now.to_f - start}" + end + end + + + +end \ No newline at end of file