diff --git a/app/models/domain_cron.rb b/app/models/domain_cron.rb index b4da056a2..dcd99328a 100644 --- a/app/models/domain_cron.rb +++ b/app/models/domain_cron.rb @@ -142,4 +142,9 @@ class DomainCron ) end + def self.delete_legal_doc_duplicates + Rake::Task['legal_doc:remove_duplicates'].reenable + Rake::Task['legal_doc:remove_duplicates'].invoke + end + end diff --git a/app/models/legal_document.rb b/app/models/legal_document.rb index b4bf3c96b..0c08958ae 100644 --- a/app/models/legal_document.rb +++ b/app/models/legal_document.rb @@ -1,4 +1,5 @@ class LegalDocument < ActiveRecord::Base + cattr_accessor :explicitly_write_file include EppErrors MIN_BODY_SIZE = (1.37 * 3.kilobytes).ceil @@ -16,7 +17,7 @@ class LegalDocument < ActiveRecord::Base validate :val_body_length, if: ->(file){ file.path.blank? && !Rails.env.staging?} before_create :add_creator - before_save :save_to_filesystem + before_save :save_to_filesystem, if: :body def epp_code_map { @@ -32,22 +33,82 @@ class LegalDocument < ActiveRecord::Base def save_to_filesystem - loop do - rand = SecureRandom.random_number.to_s.last(4) - next if rand.to_i == 0 || rand.length < 4 + binary = Base64.decode64(body) + digest = Digest::SHA1.new.update(binary).to_s - dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}" - FileUtils.mkdir_p(dir, mode: 0775) - self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}" - break unless File.file?(path) + loop do + rand = SecureRandom.random_number.to_s.last(4) + next if rand.to_i == 0 || rand.length < 4 + dir = "#{ENV['legal_documents_dir']}/#{Time.zone.now.strftime('%Y/%m/%d')}" + FileUtils.mkdir_p(dir, mode: 0775) + self.path = "#{dir}/#{Time.zone.now.to_formatted_s(:number)}_#{rand}.#{document_type}" + break unless File.file?(path) end - File.open(path, 'wb') { |f| f.write(Base64.decode64(body)) } unless Rails.env.test? + File.open(path, 'wb') { |f| f.write(binary) } if !Rails.env.test? || self.class.explicitly_write_file self.path = path + self.checksum = digest + end + + def calc_checksum + digest = Digest::SHA1.new + digest.update File.binread(path) + digest.to_s end def add_creator self.creator_str = ::PaperTrail.whodunnit true end + + + def self.remove_duplicates + start = Time.zone.now.to_f + puts '-----> Removing legal documents duplicates' + count = 0 + modified = Array.new + + LegalDocument.where(documentable_type: "Domain").where.not(checksum: [nil, ""]).find_each do |orig_legal| + next if modified.include?(orig_legal.checksum) + next if !File.exist?(orig_legal.path) + modified.push(orig_legal.checksum) + + LegalDocument.where(documentable_type: "Domain", documentable_id: orig_legal.documentable_id). + where(checksum: orig_legal.checksum). + where.not(id: orig_legal.id).where.not(path: orig_legal.path).each do |new_legal| + unless modified.include?(orig_legal.id) + File.delete(new_legal.path) if File.exist?(new_legal.path) + new_legal.update(path: orig_legal.path) + count += 1 + puts "File #{new_legal.path} has been removed by Domain #{new_legal.documentable_id}. Document id: #{new_legal.id}" + end + end + + contact_ids = DomainVersion.where(item_id: orig_legal.documentable_id).distinct. + pluck("object->>'registrant_id'", "object_changes->>'registrant_id'", + "children->>'tech_contacts'", "children->>'admin_contacts'", + "tech_contact_ids", "admin_contact_ids").flatten.uniq + contact_ids = contact_ids.map{|id| + case id + when Hash + id["id"] + when String + JSON.parse(id) rescue id.to_i + else + id + end + }.flatten.compact.uniq + LegalDocument.where(documentable_type: "Contact", documentable_id: contact_ids). + where(checksum: orig_legal.checksum).where.not(path: orig_legal.path).each do |new_legal| + unless modified.include?(orig_legal.id) + File.delete(new_legal.path) if File.exist?(new_legal.path) + new_legal.update(path: orig_legal.path) + count += 1 + puts "File #{new_legal.path} has been removed by Contact #{new_legal.documentable_id}. Document id: #{new_legal.id}" + end + end + end + puts "-----> Duplicates fixed for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds" + + end end diff --git a/db/migrate/20160629114503_add_hash_to_legal_doc.rb b/db/migrate/20160629114503_add_hash_to_legal_doc.rb new file mode 100644 index 000000000..8ea2f182d --- /dev/null +++ b/db/migrate/20160629114503_add_hash_to_legal_doc.rb @@ -0,0 +1,6 @@ +class AddHashToLegalDoc < ActiveRecord::Migration + def change + add_column :legal_documents, :checksum, :string + add_index :legal_documents, :checksum + end +end diff --git a/db/schema-read-only.rb b/db/schema-read-only.rb index 2d410461c..c633d4b60 100644 --- a/db/schema-read-only.rb +++ b/db/schema-read-only.rb @@ -11,12 +11,12 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160304125933) do +ActiveRecord::Schema.define(version: 20160629114503) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" - enable_extension "hstore" enable_extension "btree_gist" + enable_extension "hstore" create_table "account_activities", force: :cascade do |t| t.integer "account_id" @@ -214,11 +214,13 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.string "country_code" t.string "state" t.integer "legacy_id" - t.string "statuses", array: true + t.string "statuses", default: [], array: true t.hstore "status_notes" t.integer "legacy_history_id" t.integer "copy_from_id" t.datetime "ident_updated_at" + t.integer "upid" + t.datetime "up_date" end add_index "contacts", ["code"], name: "index_contacts_on_code", using: :btree @@ -261,6 +263,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "invoice_number" + t.text "request" end add_index "directos", ["item_type", "item_id"], name: "index_directos_on_item_type_and_item_id", using: :btree @@ -360,6 +363,8 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.boolean "reserved", default: false t.hstore "status_notes" t.string "statuses_backup", default: [], array: true + t.integer "upid" + t.datetime "up_date" end add_index "domains", ["delete_at"], name: "index_domains_on_delete_at", using: :btree @@ -474,8 +479,10 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "creator_str" t.string "path" + t.string "checksum" end + add_index "legal_documents", ["checksum"], name: "index_legal_documents_on_checksum", using: :btree add_index "legal_documents", ["documentable_type", "documentable_id"], name: "index_legal_documents_on_documentable_type_and_documentable_id", using: :btree create_table "log_account_activities", force: :cascade do |t| @@ -488,6 +495,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_account_activities", ["item_type", "item_id"], name: "index_log_account_activities_on_item_type_and_item_id", using: :btree @@ -503,6 +511,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_accounts", ["item_type", "item_id"], name: "index_log_accounts_on_item_type_and_item_id", using: :btree @@ -518,6 +527,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_addresses", ["item_type", "item_id"], name: "index_log_addresses_on_item_type_and_item_id", using: :btree @@ -533,6 +543,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_api_users", ["item_type", "item_id"], name: "index_log_api_users_on_item_type_and_item_id", using: :btree @@ -548,6 +559,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_bank_statements", ["item_type", "item_id"], name: "index_log_bank_statements_on_item_type_and_item_id", using: :btree @@ -563,6 +575,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_bank_transactions", ["item_type", "item_id"], name: "index_log_bank_transactions_on_item_type_and_item_id", using: :btree @@ -578,6 +591,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_blocked_domains", ["item_type", "item_id"], name: "index_log_blocked_domains_on_item_type_and_item_id", using: :btree @@ -593,6 +607,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_certificates", ["item_type", "item_id"], name: "index_log_certificates_on_item_type_and_item_id", using: :btree @@ -608,6 +623,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_contact_statuses", ["item_type", "item_id"], name: "index_log_contact_statuses_on_item_type_and_item_id", using: :btree @@ -624,6 +640,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.string "session" t.json "children" t.datetime "ident_updated_at" + t.string "uuid" end add_index "log_contacts", ["item_type", "item_id"], name: "index_log_contacts_on_item_type_and_item_id", using: :btree @@ -639,6 +656,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_countries", ["item_type", "item_id"], name: "index_log_countries_on_item_type_and_item_id", using: :btree @@ -654,6 +672,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_dnskeys", ["item_type", "item_id"], name: "index_log_dnskeys_on_item_type_and_item_id", using: :btree @@ -669,6 +688,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_domain_contacts", ["item_type", "item_id"], name: "index_log_domain_contacts_on_item_type_and_item_id", using: :btree @@ -684,6 +704,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_domain_statuses", ["item_type", "item_id"], name: "index_log_domain_statuses_on_item_type_and_item_id", using: :btree @@ -699,6 +720,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_domain_transfers", ["item_type", "item_id"], name: "index_log_domain_transfers_on_item_type_and_item_id", using: :btree @@ -717,6 +739,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.text "admin_contact_ids", default: [], array: true t.string "session" t.json "children" + t.string "uuid" end add_index "log_domains", ["item_type", "item_id"], name: "index_log_domains_on_item_type_and_item_id", using: :btree @@ -732,6 +755,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_invoice_items", ["item_type", "item_id"], name: "index_log_invoice_items_on_item_type_and_item_id", using: :btree @@ -747,6 +771,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_invoices", ["item_type", "item_id"], name: "index_log_invoices_on_item_type_and_item_id", using: :btree @@ -762,6 +787,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_keyrelays", ["item_type", "item_id"], name: "index_log_keyrelays_on_item_type_and_item_id", using: :btree @@ -777,6 +803,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_messages", ["item_type", "item_id"], name: "index_log_messages_on_item_type_and_item_id", using: :btree @@ -792,6 +819,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_nameservers", ["item_type", "item_id"], name: "index_log_nameservers_on_item_type_and_item_id", using: :btree @@ -806,6 +834,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.json "object_changes" t.datetime "created_at" t.string "session" + t.string "uuid" end create_table "log_registrars", force: :cascade do |t| @@ -818,6 +847,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_registrars", ["item_type", "item_id"], name: "index_log_registrars_on_item_type_and_item_id", using: :btree @@ -833,6 +863,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_reserved_domains", ["item_type", "item_id"], name: "index_log_reserved_domains_on_item_type_and_item_id", using: :btree @@ -848,6 +879,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_settings", ["item_type", "item_id"], name: "index_log_settings_on_item_type_and_item_id", using: :btree @@ -863,6 +895,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_users", ["item_type", "item_id"], name: "index_log_users_on_item_type_and_item_id", using: :btree @@ -878,6 +911,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end create_table "log_zonefile_settings", force: :cascade do |t| @@ -890,6 +924,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.datetime "created_at" t.string "session" t.json "children" + t.string "uuid" end add_index "log_zonefile_settings", ["item_type", "item_id"], name: "index_log_zonefile_settings_on_item_type_and_item_id", using: :btree @@ -931,6 +966,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.string "creator_str" t.string "updator_str" t.integer "legacy_domain_id" + t.string "hostname_puny" end add_index "nameservers", ["domain_id"], name: "index_nameservers_on_domain_id", using: :btree @@ -1015,6 +1051,7 @@ ActiveRecord::Schema.define(version: 20160304125933) do t.boolean "vat" t.integer "legacy_id" t.string "reference_no" + t.boolean "test_registrar", default: false end add_index "registrars", ["code"], name: "index_registrars_on_code", using: :btree diff --git a/db/structure.sql b/db/structure.sql index 78228b693..30e55f139 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -744,11 +744,13 @@ CREATE TABLE contacts ( country_code character varying, state character varying, legacy_id integer, - statuses character varying[], + statuses character varying[] DEFAULT '{}'::character varying[], status_notes hstore, legacy_history_id integer, copy_from_id integer, - ident_updated_at timestamp without time zone + ident_updated_at timestamp without time zone, + upid integer, + up_date timestamp without time zone ); @@ -887,7 +889,8 @@ CREATE TABLE directos ( response json, created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, - invoice_number character varying + invoice_number character varying, + request text ); @@ -1098,7 +1101,9 @@ CREATE TABLE domains ( statuses character varying[], reserved boolean DEFAULT false, status_notes hstore, - statuses_backup character varying[] DEFAULT '{}'::character varying[] + statuses_backup character varying[] DEFAULT '{}'::character varying[], + upid integer, + up_date timestamp without time zone ); @@ -1316,7 +1321,8 @@ CREATE TABLE legal_documents ( documentable_type character varying, created_at timestamp without time zone, creator_str character varying, - path character varying + path character varying, + checksum character varying ); @@ -1353,7 +1359,8 @@ CREATE TABLE log_account_activities ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1390,7 +1397,8 @@ CREATE TABLE log_accounts ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1427,7 +1435,8 @@ CREATE TABLE log_addresses ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1464,7 +1473,8 @@ CREATE TABLE log_api_users ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1501,7 +1511,8 @@ CREATE TABLE log_bank_statements ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1538,7 +1549,8 @@ CREATE TABLE log_bank_transactions ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1575,7 +1587,8 @@ CREATE TABLE log_blocked_domains ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1612,7 +1625,8 @@ CREATE TABLE log_certificates ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1649,7 +1663,8 @@ CREATE TABLE log_contact_statuses ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1687,7 +1702,8 @@ CREATE TABLE log_contacts ( created_at timestamp without time zone, session character varying, children json, - ident_updated_at timestamp without time zone + ident_updated_at timestamp without time zone, + uuid character varying ); @@ -1724,7 +1740,8 @@ CREATE TABLE log_countries ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1761,7 +1778,8 @@ CREATE TABLE log_dnskeys ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1798,7 +1816,8 @@ CREATE TABLE log_domain_contacts ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1835,7 +1854,8 @@ CREATE TABLE log_domain_statuses ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1872,7 +1892,8 @@ CREATE TABLE log_domain_transfers ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1912,7 +1933,8 @@ CREATE TABLE log_domains ( tech_contact_ids text[] DEFAULT '{}'::text[], admin_contact_ids text[] DEFAULT '{}'::text[], session character varying, - children json + children json, + uuid character varying ); @@ -1949,7 +1971,8 @@ CREATE TABLE log_invoice_items ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -1986,7 +2009,8 @@ CREATE TABLE log_invoices ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2023,7 +2047,8 @@ CREATE TABLE log_keyrelays ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2060,7 +2085,8 @@ CREATE TABLE log_messages ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2097,7 +2123,8 @@ CREATE TABLE log_nameservers ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2133,7 +2160,8 @@ CREATE TABLE log_pricelists ( object json, object_changes json, created_at timestamp without time zone, - session character varying + session character varying, + uuid character varying ); @@ -2170,7 +2198,8 @@ CREATE TABLE log_registrars ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2207,7 +2236,8 @@ CREATE TABLE log_reserved_domains ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2244,7 +2274,8 @@ CREATE TABLE log_settings ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2281,7 +2312,8 @@ CREATE TABLE log_users ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2318,7 +2350,8 @@ CREATE TABLE log_white_ips ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2355,7 +2388,8 @@ CREATE TABLE log_zonefile_settings ( object_changes json, created_at timestamp without time zone, session character varying, - children json + children json, + uuid character varying ); @@ -2466,7 +2500,8 @@ CREATE TABLE nameservers ( domain_id integer, creator_str character varying, updator_str character varying, - legacy_domain_id integer + legacy_domain_id integer, + hostname_puny character varying ); @@ -2673,7 +2708,8 @@ CREATE TABLE registrars ( directo_handle character varying, vat boolean, legacy_id integer, - reference_no character varying + reference_no character varying, + test_registrar boolean DEFAULT false ); @@ -4274,6 +4310,13 @@ CREATE INDEX index_keyrelays_on_domain_id ON keyrelays USING btree (domain_id); CREATE INDEX index_keyrelays_on_requester_id ON keyrelays USING btree (requester_id); +-- +-- Name: index_legal_documents_on_checksum; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- + +CREATE INDEX index_legal_documents_on_checksum ON legal_documents USING btree (checksum); + + -- -- Name: index_legal_documents_on_documentable_type_and_documentable_id; Type: INDEX; Schema: public; Owner: -; Tablespace: -- @@ -5214,5 +5257,27 @@ INSERT INTO schema_migrations (version) VALUES ('20160118092454'); INSERT INTO schema_migrations (version) VALUES ('20160218102355'); +INSERT INTO schema_migrations (version) VALUES ('20160225113801'); + +INSERT INTO schema_migrations (version) VALUES ('20160225113812'); + +INSERT INTO schema_migrations (version) VALUES ('20160226132045'); + +INSERT INTO schema_migrations (version) VALUES ('20160226132056'); + INSERT INTO schema_migrations (version) VALUES ('20160304125933'); +INSERT INTO schema_migrations (version) VALUES ('20160311085957'); + +INSERT INTO schema_migrations (version) VALUES ('20160405131315'); + +INSERT INTO schema_migrations (version) VALUES ('20160411140719'); + +INSERT INTO schema_migrations (version) VALUES ('20160414110443'); + +INSERT INTO schema_migrations (version) VALUES ('20160429114732'); + +INSERT INTO schema_migrations (version) VALUES ('20160527110738'); + +INSERT INTO schema_migrations (version) VALUES ('20160629114503'); + diff --git a/lib/tasks/legal_doc.rake b/lib/tasks/legal_doc.rake new file mode 100644 index 000000000..ad7df0fd2 --- /dev/null +++ b/lib/tasks/legal_doc.rake @@ -0,0 +1,35 @@ +namespace :legal_doc do + + desc 'Legal documents duplicates fix' + task all: :environment do + Rake::Task['legal_doc:generate_hash'].invoke + Rake::Task['legal_doc:remove_duplicates'].invoke + end + + desc 'Generate hash' + task generate_hash: :environment do + start = Time.zone.now.to_f + puts '-----> Generating unique hash for legal documents' + count = 0 + + LegalDocument.where(checksum: [nil, ""]).find_each do |x| + if File.exist?(x.path) + x.checksum = x.calc_checksum + x.save + count += 1 + end + + end + puts "-----> Hash generated for #{count} rows in #{(Time.zone.now.to_f - start).round(2)} seconds" + end + + + # Starting point is Domain legal docs + # then inside it checking the same domains and connected contacts + desc 'Remove duplicates' + task remove_duplicates: :environment do + LegalDocument.remove_duplicates + end + +end + diff --git a/spec/models/legal_documents_spec.rb b/spec/models/legal_documents_spec.rb new file mode 100644 index 000000000..e411c923d --- /dev/null +++ b/spec/models/legal_documents_spec.rb @@ -0,0 +1,71 @@ +require 'rails_helper' + +describe LegalDocument do + context 'tasks' do + it 'make files uniq' do + Fabricate(:zonefile_setting, origin: 'ee') + Fabricate(:zonefile_setting, origin: 'pri.ee') + Fabricate(:zonefile_setting, origin: 'med.ee') + Fabricate(:zonefile_setting, origin: 'fie.ee') + Fabricate(:zonefile_setting, origin: 'com.ee') + LegalDocument.explicitly_write_file = true + PaperTrail.enabled = true + + domain = Fabricate(:domain) + domain2 = Fabricate(:domain) + legals = [] + legals << original = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << copy = domain.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_different_domain = domain2.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_different = domain.legal_documents.create!(body: Base64.encode64('D' * 4.kilobytes)) + legals << skipping_as_no_checksum = domain.legal_documents.create!(checksum: nil, body: Base64.encode64('S' * 4.kilobytes)) + legals << skipping_as_no_checksum2 = domain.legal_documents.create!(checksum: "", body: Base64.encode64('S' * 4.kilobytes)) + legals << registrant_copy = domain.registrant.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << registrant_skipping_as_different = domain.registrant.legal_documents.create!(body: Base64.encode64('Q' * 4.kilobytes)) + legals << tech_copy = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << tech_skipping_as_different = domain.tech_contacts.first.legal_documents.create!(body: Base64.encode64('W' * 4.kilobytes)) + legals << admin_copy = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + legals << admin_skipping_as_different = domain.admin_contacts.first.legal_documents.create!(body: Base64.encode64('E' * 4.kilobytes)) + legals << new_second_tech_contact = domain2.tech_contacts.first.legal_documents.create!(body: Base64.encode64('S' * 4.kilobytes)) + domain.tech_contacts << domain2.tech_contacts.first + + + # writing nesting to history + domain.update(updated_at: Time.now) + domain2.update(updated_at: Time.now) + domain.reload + + skipping_as_no_checksum.update_columns(checksum: nil) + skipping_as_no_checksum2.update_columns(checksum: "") + skipping_as_no_checksum.reload + skipping_as_no_checksum2.reload + skipping_as_no_checksum.path.should_not == skipping_as_no_checksum2.path + + skipping_as_no_checksum.checksum.should == nil + skipping_as_no_checksum2.checksum.should == "" + original.checksum.should == copy.checksum + original.checksum.should_not == skipping_as_different.checksum + domain.tech_contacts.count.should == 2 + + LegalDocument.remove_duplicates + LegalDocument.remove_duplicates + LegalDocument.remove_duplicates + legals.each(&:reload) + + skipping_as_no_checksum.path.should_not be(skipping_as_no_checksum2.path) + original.path.should_not == skipping_as_different.path + original.path.should_not == skipping_as_different_domain.path + original.path.should_not == registrant_skipping_as_different.path + original.path.should_not == tech_skipping_as_different.path + original.path.should_not == admin_skipping_as_different.path + original.path.should == copy.path + original.path.should == registrant_copy.path + original.path.should == tech_copy.path + original.path.should == admin_copy.path + + original.path.should == new_second_tech_contact.path + skipping_as_different_domain.path.should_not == new_second_tech_contact.path + end + end + +end \ No newline at end of file