From 6b0e155bc111eb52a50214a945775baf51ad8bf4 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 10:16:32 +0300 Subject: [PATCH 01/29] Handle domain whois body in sync --- app/models/domain.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/domain.rb b/app/models/domain.rb index 36092ed90..8222cea76 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -265,7 +265,7 @@ class Domain < ActiveRecord::Base update_column(whois_body: new_whois_body) update_whois_server end - handle_asynchronously :update_whois_body + # handle_asynchronously :update_whois_body # rubocop:enable Metrics/MethodLength def contacts_body From 7cb0a4bdd39003affa0ea1a3d9bebe1aa6ef1e45 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Thu, 26 Feb 2015 18:04:24 +0200 Subject: [PATCH 02/29] Registrars import --- Gemfile | 4 +- Gemfile.lock | 65 +++++++++---------- app/models/legacy/db.rb | 10 +++ app/models/legacy/registrar.rb | 5 ++ ...0150226121252_remove_country_id_columns.rb | 7 ++ ...144723_add_legacy_columns_for_registrar.rb | 8 +++ db/schema.rb | 8 +-- lib/tasks/import.rake | 36 ++++++++++ 8 files changed, 105 insertions(+), 38 deletions(-) create mode 100644 app/models/legacy/db.rb create mode 100644 app/models/legacy/registrar.rb create mode 100644 db/migrate/20150226121252_remove_country_id_columns.rb create mode 100644 db/migrate/20150226144723_add_legacy_columns_for_registrar.rb create mode 100644 lib/tasks/import.rake diff --git a/Gemfile b/Gemfile index fc26d9d02..85f348462 100644 --- a/Gemfile +++ b/Gemfile @@ -71,6 +71,9 @@ gem 'depp', github: 'domify/depp', ref: '800ab30dcb6dae33095bcb9df47b4e0a390891a # gem 'depp', path: '~/projects/depp' gem 'epp', '~> 1.4.2', github: 'gitlabeu/epp' +# for importing legacy db +gem 'activerecord-import', '~> 0.7.0' # for inserting dummy data + group :development do # dev tools gem 'spring', '~> 1.3.3' @@ -102,7 +105,6 @@ group :development, :test do gem 'launchy', '~> 2.4.3' # for opening browser automatically # helper gems - gem 'activerecord-import', '~> 0.6.0' # for inserting dummy data gem 'database_cleaner', '~> 1.3.0' # For cleaning db in feature and epp tests gem 'faker', '~> 1.3.0' # Library to generate fake data diff --git a/Gemfile.lock b/Gemfile.lock index 40dce6b12..f92ac9739 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,7 +79,7 @@ GEM activemodel (= 4.2.1) activesupport (= 4.2.1) arel (~> 6.0) - activerecord-import (0.6.0) + activerecord-import (0.7.0) activerecord (>= 3.0) activesupport (4.2.1) i18n (~> 0.7) @@ -87,15 +87,15 @@ GEM minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - addressable (2.3.6) - akami (1.2.2) + addressable (2.3.8) + akami (1.3.0) gyoku (>= 0.4.0) nokogiri arel (6.0.0) ast (2.0.0) astrolabe (1.3.0) parser (>= 2.2.0.pre.3, < 3.0) - autodoc (0.4.3) + autodoc (0.4.4) actionpack activesupport (>= 3.0.0) rspec @@ -103,7 +103,7 @@ GEM descendants_tracker (~> 0.0.4) ice_nine (~> 0.11.0) thread_safe (~> 0.3, >= 0.3.1) - bcrypt (3.1.9) + bcrypt (3.1.10) better_errors (2.0.0) coderay (>= 1.0.0) erubis (>= 2.6.6) @@ -186,14 +186,14 @@ GEM epp-xml (0.10.4) activesupport (~> 4.1) builder (~> 3.2) - equalizer (0.0.9) + equalizer (0.0.11) erubis (2.7.0) execjs (2.4.0) fabrication (2.12.2) faker (1.3.0) i18n (~> 0.5) fastercsv (1.5.5) - ffi (1.9.6) + ffi (1.9.8) figaro (1.1.0) thor (~> 0.14) flay (2.4.0) @@ -229,7 +229,7 @@ GEM guard-rubocop (1.1.0) guard (~> 2.0) rubocop (~> 0.20) - gyoku (1.2.3) + gyoku (1.3.0) builder (>= 2.1.2) haml (4.0.6) tilt @@ -239,22 +239,21 @@ GEM haml (>= 4.0.6, < 5.0) html2haml (>= 1.0.1) railties (>= 4.0.1) - hashie (3.3.2) - hashie_rails (0.0.1) + hashie (3.4.1) + hashie_rails (0.0.4) hashie (>= 3.0) - rails (~> 4.0) highline (1.6.21) hike (1.2.3) hitimes (1.2.2) hpricot (0.8.6) httpclient (2.6.0.1) - httpi (2.3.0) + httpi (2.4.0) rack i18n (0.7.0) ice_nine (0.11.1) isikukood (0.1.2) - iso8601 (0.8.2) - jbuilder (2.2.6) + iso8601 (0.8.5) + jbuilder (2.2.12) activesupport (>= 3.0.0, < 5) multi_json (~> 1.2) jquery-rails (4.0.3) @@ -270,8 +269,8 @@ GEM addressable (~> 2.3) libv8 (3.16.14.7) libxml-ruby (2.8.0) - listen (2.8.5) - celluloid (>= 0.15.2) + listen (2.10.0) + celluloid (~> 0.16.0) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) loofah (2.0.1) @@ -283,7 +282,7 @@ GEM mime-types (>= 1.16, < 3) method_source (0.8.2) mime-types (2.4.3) - mina (0.3.1) + mina (0.3.4) open4 (~> 1.3.4) rake mini_portile (0.6.2) @@ -293,11 +292,11 @@ GEM newrelic_rpm (3.9.9.275) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) - nori (2.4.0) - nprogress-rails (0.1.6.5) + nori (2.5.0) + nprogress-rails (0.1.6.6) open4 (1.3.4) orm_adapter (0.5.0) - parser (2.2.0.2) + parser (2.2.0.3) ast (>= 1.1, < 3.0) pg (0.18.1) phantomjs (1.9.7.1) @@ -369,8 +368,8 @@ GEM sexp_processor ref (1.0.5) request_store (1.1.0) - responders (2.0.2) - railties (>= 4.2.0.alpha, < 5) + responders (2.1.0) + railties (>= 4.2.0, < 5) rspec (3.0.0) rspec-core (~> 3.0.0) rspec-expectations (~> 3.0.0) @@ -397,7 +396,7 @@ GEM powerpack (~> 0.0.6) rainbow (>= 1.99.1, < 3.0) ruby-progressbar (~> 1.4) - ruby-progressbar (1.7.1) + ruby-progressbar (1.7.5) ruby2ruby (2.1.3) ruby_parser (~> 3.1) sexp_processor (~> 4.0) @@ -412,13 +411,13 @@ GEM virtus (~> 1.0) safe_yaml (1.0.4) sass (3.4.13) - sass-rails (5.0.1) + sass-rails (5.0.2) railties (>= 4.0.0, < 5.0) sass (~> 3.1) sprockets (>= 2.8, < 4.0) sprockets-rails (>= 2.0, < 4.0) tilt (~> 1.1) - savon (2.10.1) + savon (2.11.0) akami (~> 1.2) builder (>= 2.1.2) gyoku (~> 1.2) @@ -434,11 +433,11 @@ GEM sexp_processor (4.5.0) shoulda-matchers (2.6.2) activesupport (>= 3.0.0) - simplecov (0.9.1) + simplecov (0.9.2) docile (~> 1.1.0) multi_json (~> 1.0) - simplecov-html (~> 0.8.0) - simplecov-html (0.8.0) + simplecov-html (~> 0.9.0) + simplecov-html (0.9.0) simpleidn (0.0.5) slim (2.1.0) temple (~> 0.6.9) @@ -486,19 +485,19 @@ GEM uuid (2.3.7) macaddr (~> 1.0) uuidtools (2.1.5) - virtus (1.0.4) + virtus (1.0.5) axiom-types (~> 0.1) coercible (~> 1.0) descendants_tracker (~> 0.0, >= 0.0.3) equalizer (~> 0.0, >= 0.0.9) warden (1.2.3) rack (>= 1.0) - wasabi (3.4.0) + wasabi (3.5.0) httpi (~> 2.0) nokogiri (>= 1.4.2) - websocket-driver (0.5.1) + websocket-driver (0.5.4) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.1) + websocket-extensions (0.1.2) whenever (0.9.4) chronic (>= 0.6.3) xpath (2.0.0) @@ -508,7 +507,7 @@ PLATFORMS ruby DEPENDENCIES - activerecord-import (~> 0.6.0) + activerecord-import (~> 0.7.0) autodoc better_errors (~> 2.0.0) binding_of_caller (~> 0.7.2) diff --git a/app/models/legacy/db.rb b/app/models/legacy/db.rb new file mode 100644 index 000000000..f960683d0 --- /dev/null +++ b/app/models/legacy/db.rb @@ -0,0 +1,10 @@ +module Legacy + class Db < ActiveRecord::Base + self.abstract_class = true + establish_connection :fred + + def readonly? + true + end + end +end diff --git a/app/models/legacy/registrar.rb b/app/models/legacy/registrar.rb new file mode 100644 index 000000000..0dc124085 --- /dev/null +++ b/app/models/legacy/registrar.rb @@ -0,0 +1,5 @@ +module Legacy + class Registrar < Db + self.table_name = :registrar + end +end diff --git a/db/migrate/20150226121252_remove_country_id_columns.rb b/db/migrate/20150226121252_remove_country_id_columns.rb new file mode 100644 index 000000000..f5036f496 --- /dev/null +++ b/db/migrate/20150226121252_remove_country_id_columns.rb @@ -0,0 +1,7 @@ +class RemoveCountryIdColumns < ActiveRecord::Migration + def change + remove_column :registrars, :country_id, :integer + remove_column :users, :country_id, :integer + remove_column :addresses, :country_id, :integer + end +end diff --git a/db/migrate/20150226144723_add_legacy_columns_for_registrar.rb b/db/migrate/20150226144723_add_legacy_columns_for_registrar.rb new file mode 100644 index 000000000..212e882bc --- /dev/null +++ b/db/migrate/20150226144723_add_legacy_columns_for_registrar.rb @@ -0,0 +1,8 @@ +class AddLegacyColumnsForRegistrar < ActiveRecord::Migration + def change + add_column :registrars, :url, :string + add_column :registrars, :directo_handle, :string + add_column :registrars, :vat, :boolean + add_column :registrars, :legacy_id, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 23f93762c..19104f421 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -18,7 +18,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do create_table "addresses", force: :cascade do |t| t.integer "contact_id" - t.integer "country_id" t.string "city" t.string "street" t.string "zip" @@ -579,7 +578,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "name" t.string "reg_no" t.string "vat_no" - t.integer "country_id" t.string "billing_address" t.datetime "created_at" t.datetime "updated_at" @@ -594,6 +592,10 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "street" t.string "zip" t.string "code" + t.string "url" + t.string "directo_handle" + t.boolean "vat" + t.integer "legacy_id" end add_index "registrars", ["code"], name: "index_registrars_on_code", using: :btree @@ -631,8 +633,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.inet "current_sign_in_ip" t.inet "last_sign_in_ip" t.string "identity_code" - t.integer "country_id" - t.string "roles", array: true t.string "creator_str" t.string "updator_str" t.string "country_code" diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 000000000..5ddd26ece --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,36 @@ +namespace :import do + desc "Imports registrars" + task registrars: :environment do + puts '-----> Importing registrars...' + + # Registrar.where('legacy_id IS NOT NULL').delete_all + + registrars = [] + Legacy::Registrar.all.each do |x| + next if Registrar.exists?(legacy_id: x.id) + + registrars << Registrar.new({ + name: x.organization.try(:strip).presence || x.name.try(:strip).presence || x.handle.try(:strip).presence, + reg_no: x.ico.try(:strip), + vat_no: x.dic.try(:strip), + billing_address: nil, + phone: x.telephone.try(:strip), + email: x.email.try(:strip), + billing_email: x.billing_address.try(:strip), + country_code: x.country.try(:strip), + state: x.stateorprovince.try(:strip), + city: x.city.try(:strip), + street: x.street1.try(:strip), + zip: x.postalcode.try(:strip), + url: x.url.try(:strip), + directo_handle: x.directo_handle.try(:strip), + vat: x.vat, + legacy_id: x.id + }) + end + + Registrar.import registrars, validate: false + + puts '-----> Registrars imported' + end +end From 3149d0731d3547a66229b5964ef280ff0c624cc8 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Fri, 27 Feb 2015 10:52:41 +0200 Subject: [PATCH 03/29] Optimize registrars import --- lib/tasks/import.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 5ddd26ece..426c41939 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -6,8 +6,10 @@ namespace :import do # Registrar.where('legacy_id IS NOT NULL').delete_all registrars = [] + existing_ids = Registrar.pluck(:legacy_id) + Legacy::Registrar.all.each do |x| - next if Registrar.exists?(legacy_id: x.id) + next if existing_ids.include?(x.id) registrars << Registrar.new({ name: x.organization.try(:strip).presence || x.name.try(:strip).presence || x.handle.try(:strip).presence, From a598b46723df575038649e3b882d00ad839553d7 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Fri, 27 Feb 2015 12:15:28 +0200 Subject: [PATCH 04/29] WIP with contact import --- app/models/legacy/contact.rb | 5 ++ app/models/legacy/object_registry.rb | 6 +++ ...27092508_add_legacy_columns_for_contact.rb | 5 ++ lib/tasks/import.rake | 50 +++++++++++++++++-- 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 app/models/legacy/contact.rb create mode 100644 app/models/legacy/object_registry.rb create mode 100644 db/migrate/20150227092508_add_legacy_columns_for_contact.rb diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb new file mode 100644 index 000000000..64da7c886 --- /dev/null +++ b/app/models/legacy/contact.rb @@ -0,0 +1,5 @@ +module Legacy + class Contact < Db + self.table_name = :contact + end +end diff --git a/app/models/legacy/object_registry.rb b/app/models/legacy/object_registry.rb new file mode 100644 index 000000000..bb0f003e4 --- /dev/null +++ b/app/models/legacy/object_registry.rb @@ -0,0 +1,6 @@ +module Legacy + class ObjectRegistry < Db + self.table_name = :object_registry + self.inheritance_column = nil + end +end diff --git a/db/migrate/20150227092508_add_legacy_columns_for_contact.rb b/db/migrate/20150227092508_add_legacy_columns_for_contact.rb new file mode 100644 index 000000000..0aba53c38 --- /dev/null +++ b/db/migrate/20150227092508_add_legacy_columns_for_contact.rb @@ -0,0 +1,5 @@ +class AddLegacyColumnsForContact < ActiveRecord::Migration + def change + add_column :contacts, :legacy_id, :integer + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 426c41939..754f1b708 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,9 +1,9 @@ namespace :import do - desc "Imports registrars" + desc 'Imports registrars' task registrars: :environment do puts '-----> Importing registrars...' - # Registrar.where('legacy_id IS NOT NULL').delete_all + Registrar.where('legacy_id IS NOT NULL').delete_all registrars = [] existing_ids = Registrar.pluck(:legacy_id) @@ -27,7 +27,9 @@ namespace :import do url: x.url.try(:strip), directo_handle: x.directo_handle.try(:strip), vat: x.vat, - legacy_id: x.id + legacy_id: x.id, + creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}" }) end @@ -35,4 +37,46 @@ namespace :import do puts '-----> Registrars imported' end + + # desc 'Import contacts' + # task contact: :environment do + # puts '-----> Importing contacts...' + + # contacts = [] + # existing_ids = Contact.pluck(:legacy_id) + + # Legacy::Contact.all.each do |x| + # next if existing_ids.include?(x.id) + + # 1;"RC";"born number" + # 2;"OP";"identity card number" -> priv + # 3;"PASS";"passwport" -> + # 4;"ICO";"organization identification number" + # 5;"MPSV";"social system identification" + # 6;"BIRTHDAY";"day of birth" + + # contacts << Contact.new({ + # code: , + # #type: , # not needed + # #reg_no: x.ssn.try(:strip), + # phone: x.telephone.try(:strip), + # email: x.email.try(:strip), + # fax: x.fax.try(:strip), + # ident: x.ssn.try(:strip), + # ident_type: , + # #created_by_id: , # not needed + # #updated_by_id: , # not needed + # auth_info: , + # name: x.name.try(:strip), + # org_name: x.organization.try(:strip), + # registrar_id: , + # creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + # updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}" + # ident_country_code: x.country.try(:strip), + # legacy_id: x.id + # }) + # end + + # puts '-----> Contacts imported' + # end end From 9cd4bad3a9da13f57d36b1d1f1e47fff750c8440 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Fri, 27 Feb 2015 18:13:03 +0200 Subject: [PATCH 05/29] Map legacy contact fields to contact --- app/models/legacy/contact.rb | 2 + app/models/legacy/object.rb | 10 ++ ...27092508_add_legacy_columns_for_contact.rb | 4 + db/schema.rb | 5 +- lib/tasks/import.rake | 96 ++++++++++++------- 5 files changed, 77 insertions(+), 40 deletions(-) create mode 100644 app/models/legacy/object.rb diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 64da7c886..0951d9df8 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -1,5 +1,7 @@ module Legacy class Contact < Db self.table_name = :contact + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id end end diff --git a/app/models/legacy/object.rb b/app/models/legacy/object.rb new file mode 100644 index 000000000..dc1c53d7b --- /dev/null +++ b/app/models/legacy/object.rb @@ -0,0 +1,10 @@ +module Legacy + class Object < Db + self.table_name = :object + + def self.instance_method_already_implemented?(method_name) + return true if method_name == 'update' + super + end + end +end diff --git a/db/migrate/20150227092508_add_legacy_columns_for_contact.rb b/db/migrate/20150227092508_add_legacy_columns_for_contact.rb index 0aba53c38..2d08e3094 100644 --- a/db/migrate/20150227092508_add_legacy_columns_for_contact.rb +++ b/db/migrate/20150227092508_add_legacy_columns_for_contact.rb @@ -1,5 +1,9 @@ class AddLegacyColumnsForContact < ActiveRecord::Migration def change add_column :contacts, :legacy_id, :integer + remove_column :contacts, :type, :string + remove_column :contacts, :reg_no, :string + remove_column :contacts, :created_by_id, :integer + remove_column :contacts, :updated_by_id, :integer end end diff --git a/db/schema.rb b/db/schema.rb index 19104f421..3242725ac 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -74,8 +74,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do create_table "contacts", force: :cascade do |t| t.string "code" - t.string "type" - t.string "reg_no" t.string "phone" t.string "email" t.string "fax" @@ -83,8 +81,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.datetime "updated_at" t.string "ident" t.string "ident_type" - t.integer "created_by_id" - t.integer "updated_by_id" t.string "auth_info" t.string "name" t.string "org_name" @@ -97,6 +93,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "zip" t.string "country_code" t.string "state" + t.integer "legacy_id" end add_index "contacts", ["code"], name: "index_contacts_on_code", using: :btree diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 754f1b708..25f455ea4 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -38,45 +38,69 @@ namespace :import do puts '-----> Registrars imported' end - # desc 'Import contacts' - # task contact: :environment do - # puts '-----> Importing contacts...' + desc 'Import contacts' + task contacts: :environment do + puts '-----> Importing contacts...' - # contacts = [] - # existing_ids = Contact.pluck(:legacy_id) + contacts = [] + existing_ids = Contact.pluck(:legacy_id) - # Legacy::Contact.all.each do |x| - # next if existing_ids.include?(x.id) + # 1;"RC";"born number" # not used + # 2;"OP";"identity card number" -> priv + # 3;"PASS";"passwport" -> + # 4;"ICO";"organization identification number" + # 5;"MPSV";"social system identification" # not used + # 6;"BIRTHDAY";"day of birth" - # 1;"RC";"born number" - # 2;"OP";"identity card number" -> priv - # 3;"PASS";"passwport" -> - # 4;"ICO";"organization identification number" - # 5;"MPSV";"social system identification" - # 6;"BIRTHDAY";"day of birth" + ident_type_map = { + 2 => Contact::IDENT_PRIV, + 3 => Contact::IDENT_PASSPORT, + 4 => Contact::IDENT_TYPE_BIC, + 6 => Contact::IDENT_BIRTHDAY + } - # contacts << Contact.new({ - # code: , - # #type: , # not needed - # #reg_no: x.ssn.try(:strip), - # phone: x.telephone.try(:strip), - # email: x.email.try(:strip), - # fax: x.fax.try(:strip), - # ident: x.ssn.try(:strip), - # ident_type: , - # #created_by_id: , # not needed - # #updated_by_id: , # not needed - # auth_info: , - # name: x.name.try(:strip), - # org_name: x.organization.try(:strip), - # registrar_id: , - # creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - # updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}" - # ident_country_code: x.country.try(:strip), - # legacy_id: x.id - # }) - # end + Legacy::Contact.all.each do |x| + next if existing_ids.include?(x.id) + begin + registrar = Registrar.find_by(legacy_id: x.object_registry.crid) - # puts '-----> Contacts imported' - # end + contacts << Contact.new({ + code: x.object_registry.name, + #type: , # not needed + #reg_no: x.ssn.try(:strip), + phone: x.telephone.try(:strip), + email: x.email.try(:strip), + fax: x.fax.try(:strip), + ident: x.ssn.try(:strip), + ident_type: ident_type_map[x.ssntype], + #created_by_id: , # not needed + #updated_by_id: , # not needed + auth_info: x.object.authinfopw.try(:strip), + name: x.name.try(:strip), + org_name: x.organization.try(:strip), + registrar_id: registrar.try(:id), + creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + ident_country_code: x.country.try(:strip), + created_at: x.try(:crdate), + legacy_id: x.id, + address: Address.new({ + city: x.city.try(:strip), + street: x.street1.try(:strip), + zip: x.postalcode.try(:strip), + street2: x.street2.try(:strip), + street3: x.street3.try(:strip), + creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", + country_code: x.country.try(:strip), + state: x.stateorprovince.try(:strip) + }) + }) + rescue => e + binding.pry + end + end + + puts '-----> Contacts imported' + end end From 20f8192f785a30ec8d90b99302e5a957d28c23a3 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Mon, 2 Mar 2015 11:40:40 +0200 Subject: [PATCH 06/29] Find in batches for contact import --- app/models/legacy/contact.rb | 1 + lib/tasks/import.rake | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 0951d9df8..736666e5b 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -3,5 +3,6 @@ module Legacy self.table_name = :contact belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id + belongs_to :registrar, foreign_key: :crid end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 25f455ea4..f6e89b293 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -40,6 +40,7 @@ namespace :import do desc 'Import contacts' task contacts: :environment do + start = Time.now.to_i puts '-----> Importing contacts...' contacts = [] @@ -59,10 +60,12 @@ namespace :import do 6 => Contact::IDENT_BIRTHDAY } - Legacy::Contact.all.each do |x| + contacts = [] + + Legacy::Contact.includes(:object_registry, :object, :registrar).find_each(batch_size: 10000) do |x| next if existing_ids.include?(x.id) begin - registrar = Registrar.find_by(legacy_id: x.object_registry.crid) + # registrar = Registrar.find_by(legacy_id: x.object_registry.crid) contacts << Contact.new({ code: x.object_registry.name, @@ -78,7 +81,7 @@ namespace :import do auth_info: x.object.authinfopw.try(:strip), name: x.name.try(:strip), org_name: x.organization.try(:strip), - registrar_id: registrar.try(:id), + registrar_id: x.registrar.try(:id), creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", ident_country_code: x.country.try(:strip), @@ -96,8 +99,16 @@ namespace :import do state: x.stateorprovince.try(:strip) }) }) + + if contacts.size % 10000 == 0 + puts "commtting #{contacts.size}" + puts Time.now.to_i - start + # Contact.import contacts, validate: false + contacts = [] + end rescue => e - binding.pry + puts e + puts x.inspect end end From e195d3948e0ec6d8599ef299f3dc1fef4e111b00 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Mon, 2 Mar 2015 17:05:14 +0200 Subject: [PATCH 07/29] Contact importing --- app/models/legacy/contact.rb | 1 - app/models/legacy/object_registry.rb | 1 + ...02130224_add_legacy_columns_for_address.rb | 5 + db/schema.rb | 6 + lib/tasks/import.rake | 125 +++++++++++------- 5 files changed, 92 insertions(+), 46 deletions(-) create mode 100644 db/migrate/20150302130224_add_legacy_columns_for_address.rb diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 736666e5b..0951d9df8 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -3,6 +3,5 @@ module Legacy self.table_name = :contact belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id - belongs_to :registrar, foreign_key: :crid end end diff --git a/app/models/legacy/object_registry.rb b/app/models/legacy/object_registry.rb index bb0f003e4..c6885318c 100644 --- a/app/models/legacy/object_registry.rb +++ b/app/models/legacy/object_registry.rb @@ -2,5 +2,6 @@ module Legacy class ObjectRegistry < Db self.table_name = :object_registry self.inheritance_column = nil + belongs_to :registrar, foreign_key: :crid, primary_key: :legacy_id, class_name: '::Registrar' end end diff --git a/db/migrate/20150302130224_add_legacy_columns_for_address.rb b/db/migrate/20150302130224_add_legacy_columns_for_address.rb new file mode 100644 index 000000000..3a2d4e8ba --- /dev/null +++ b/db/migrate/20150302130224_add_legacy_columns_for_address.rb @@ -0,0 +1,5 @@ +class AddLegacyColumnsForAddress < ActiveRecord::Migration + def change + add_column :addresses, :legacy_contact_id, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 3242725ac..e808473cd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,11 @@ # # It's strongly recommended that you check this file into your version control system. +<<<<<<< HEAD ActiveRecord::Schema.define(version: 20150330083700) do +======= +ActiveRecord::Schema.define(version: 20150302130224) do +>>>>>>> Contact importing # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -29,6 +33,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "updator_str" t.string "country_code" t.string "state" + t.integer "legacy_contact_id" end create_table "api_users", force: :cascade do |t| @@ -630,6 +635,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.inet "current_sign_in_ip" t.inet "last_sign_in_ip" t.string "identity_code" + t.string "roles", array: true t.string "creator_str" t.string "updator_str" t.string "country_code" diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index f6e89b293..d98cfec49 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -43,9 +43,6 @@ namespace :import do start = Time.now.to_i puts '-----> Importing contacts...' - contacts = [] - existing_ids = Contact.pluck(:legacy_id) - # 1;"RC";"born number" # not used # 2;"OP";"identity card number" -> priv # 3;"PASS";"passwport" -> @@ -60,58 +57,96 @@ namespace :import do 6 => Contact::IDENT_BIRTHDAY } - contacts = [] + contact_columns = [ + "code", + "phone", + "email", + "fax", + "created_at", + "ident", + "ident_type", + "auth_info", + "name", + "org_name", + "registrar_id", + "creator_str", + "updator_str", + "ident_country_code", + "legacy_id" + ] + + address_columns = [ + "city", + "street", + "zip", + "street2", + "street3", + "creator_str", + "updator_str", + "country_code", + "state", + "legacy_contact_id" + ] + + contacts, addresses = [], [] + existing_contact_ids = Contact.pluck(:legacy_id) + existing_address_ids = Address.pluck(:legacy_contact_id) + user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" + count = 0 + + Legacy::Contact.includes(:object_registry, :object, :object_registry => :registrar).find_each(batch_size: 10000).with_index do |x, index| + next if existing_contact_ids.include?(x.id) + next if existing_address_ids.include?(x.id) + count += 1 - Legacy::Contact.includes(:object_registry, :object, :registrar).find_each(batch_size: 10000) do |x| - next if existing_ids.include?(x.id) begin - # registrar = Registrar.find_by(legacy_id: x.object_registry.crid) + contacts << [ + x.object_registry.name, + x.telephone.try(:strip), + x.email.try(:strip), + x.fax.try(:strip), + x.try(:crdate), + x.ssn.try(:strip), + ident_type_map[x.ssntype], + x.object.authinfopw.try(:strip), + x.name.try(:strip), + x.organization.try(:strip), + x.object_registry.try(:registrar).try(:id), + user, + user, + x.country.try(:strip), + x.id + ] - contacts << Contact.new({ - code: x.object_registry.name, - #type: , # not needed - #reg_no: x.ssn.try(:strip), - phone: x.telephone.try(:strip), - email: x.email.try(:strip), - fax: x.fax.try(:strip), - ident: x.ssn.try(:strip), - ident_type: ident_type_map[x.ssntype], - #created_by_id: , # not needed - #updated_by_id: , # not needed - auth_info: x.object.authinfopw.try(:strip), - name: x.name.try(:strip), - org_name: x.organization.try(:strip), - registrar_id: x.registrar.try(:id), - creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - ident_country_code: x.country.try(:strip), - created_at: x.try(:crdate), - legacy_id: x.id, - address: Address.new({ - city: x.city.try(:strip), - street: x.street1.try(:strip), - zip: x.postalcode.try(:strip), - street2: x.street2.try(:strip), - street3: x.street3.try(:strip), - creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - country_code: x.country.try(:strip), - state: x.stateorprovince.try(:strip) - }) - }) + addresses << [ + x.city.try(:strip), + x.street1.try(:strip), + x.postalcode.try(:strip), + x.street2.try(:strip), + x.street3.try(:strip), + user, + user, + x.country.try(:strip), + x.stateorprovince.try(:strip), + x.id + ] if contacts.size % 10000 == 0 - puts "commtting #{contacts.size}" - puts Time.now.to_i - start - # Contact.import contacts, validate: false - contacts = [] + Contact.import contact_columns, contacts, validate: false + Address.import address_columns, addresses, validate: false + contacts, addresses = [], [] end rescue => e + puts "ERROR on index #{index}" puts e - puts x.inspect end end - puts '-----> Contacts imported' + Contact.import contact_columns, contacts, validate: false + Address.import address_columns, addresses, validate: false + + puts '-----> Updating relations...' + ActiveRecord::Base.connection.execute('UPDATE addresses SET contact_id = legacy_contact_id WHERE legacy_contact_id IS NOT NULL AND contact_id IS NULL') + puts "-----> Imported #{count} new contacts in #{Time.now.to_i - start} seconds" end end From ecee4fa64cf9f3db15f29aa3283f9b4ea1174208 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Mon, 2 Mar 2015 17:17:57 +0200 Subject: [PATCH 08/29] Log timing in import script --- lib/tasks/import.rake | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index d98cfec49..ef794ae8b 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,15 +1,17 @@ namespace :import do desc 'Imports registrars' task registrars: :environment do + start = Time.now.to_f puts '-----> Importing registrars...' - Registrar.where('legacy_id IS NOT NULL').delete_all - registrars = [] existing_ids = Registrar.pluck(:legacy_id) + user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" + count = 0 Legacy::Registrar.all.each do |x| next if existing_ids.include?(x.id) + count += 1 registrars << Registrar.new({ name: x.organization.try(:strip).presence || x.name.try(:strip).presence || x.handle.try(:strip).presence, @@ -28,19 +30,19 @@ namespace :import do directo_handle: x.directo_handle.try(:strip), vat: x.vat, legacy_id: x.id, - creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", - updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}" + creator_str: user, + updator_str: user }) end Registrar.import registrars, validate: false - puts '-----> Registrars imported' + puts "-----> Imported #{count} new registrars in #{(Time.now.to_f - start).round(2)} seconds" end desc 'Import contacts' task contacts: :environment do - start = Time.now.to_i + start = Time.now.to_f puts '-----> Importing contacts...' # 1;"RC";"born number" # not used @@ -147,6 +149,6 @@ namespace :import do puts '-----> Updating relations...' ActiveRecord::Base.connection.execute('UPDATE addresses SET contact_id = legacy_contact_id WHERE legacy_contact_id IS NOT NULL AND contact_id IS NULL') - puts "-----> Imported #{count} new contacts in #{Time.now.to_i - start} seconds" + puts "-----> Imported #{count} new contacts in #{(Time.now.to_f - start).round(2)} seconds" end end From 7707f5ff8e703061c90953945ea43260cbabd3d2 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Mon, 2 Mar 2015 17:38:04 +0200 Subject: [PATCH 09/29] Fix relations sql --- lib/tasks/import.rake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index ef794ae8b..5bb18b73f 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -148,7 +148,14 @@ namespace :import do Address.import address_columns, addresses, validate: false puts '-----> Updating relations...' - ActiveRecord::Base.connection.execute('UPDATE addresses SET contact_id = legacy_contact_id WHERE legacy_contact_id IS NOT NULL AND contact_id IS NULL') + ActiveRecord::Base.connection.execute( + "UPDATE addresses "\ + "SET contact_id = contacts.id "\ + "FROM contacts "\ + "WHERE contacts.legacy_id = legacy_contact_id "\ + "AND legacy_contact_id IS NOT NULL "\ + "AND contact_id IS NULL" + ) puts "-----> Imported #{count} new contacts in #{(Time.now.to_f - start).round(2)} seconds" end end From 41b51455cf2dca33cfa76948265e08532f246e62 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Tue, 3 Mar 2015 14:53:31 +0200 Subject: [PATCH 10/29] Basic domain import --- app/models/legacy/contact.rb | 1 + app/models/legacy/domain.rb | 10 +++ app/models/legacy/enum_object_state.rb | 5 ++ app/models/legacy/object_state.rb | 9 +++ ...302161712_add_legacy_columns_for_domain.rb | 6 ++ db/schema.rb | 7 +- lib/tasks/import.rake | 79 +++++++++++++++++-- 7 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 app/models/legacy/domain.rb create mode 100644 app/models/legacy/enum_object_state.rb create mode 100644 app/models/legacy/object_state.rb create mode 100644 db/migrate/20150302161712_add_legacy_columns_for_domain.rb diff --git a/app/models/legacy/contact.rb b/app/models/legacy/contact.rb index 0951d9df8..e77c39b36 100644 --- a/app/models/legacy/contact.rb +++ b/app/models/legacy/contact.rb @@ -3,5 +3,6 @@ module Legacy self.table_name = :contact belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id + belongs_to :object_state, foreign_key: :id, primary_key: :object_id end end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb new file mode 100644 index 000000000..a28999e0e --- /dev/null +++ b/app/models/legacy/domain.rb @@ -0,0 +1,10 @@ +module Legacy + class Domain < Db + self.table_name = :domain + + belongs_to :object_registry, foreign_key: :id + belongs_to :object, foreign_key: :id + belongs_to :object_state, foreign_key: :id, primary_key: :object_id + belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' + end +end diff --git a/app/models/legacy/enum_object_state.rb b/app/models/legacy/enum_object_state.rb new file mode 100644 index 000000000..00a096890 --- /dev/null +++ b/app/models/legacy/enum_object_state.rb @@ -0,0 +1,5 @@ +module Legacy + class EnumObjectState < Db + self.table_name = :enum_object_states + end +end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb new file mode 100644 index 000000000..39767f69c --- /dev/null +++ b/app/models/legacy/object_state.rb @@ -0,0 +1,9 @@ +module Legacy + class ObjectState < Db + self.table_name = :object_state + + belongs_to :enum_object_state, foreign_key: :state_id + + delegate :name, to: :enum_object_state, prefix: false, allow_nil: true + end +end diff --git a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb new file mode 100644 index 000000000..688ae9a8e --- /dev/null +++ b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb @@ -0,0 +1,6 @@ +class AddLegacyColumnsForDomain < ActiveRecord::Migration + def change + add_column :domains, :legacy_id, :integer + add_column :nameservers, :legacy_id, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index e808473cd..ec477ebd7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,12 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -<<<<<<< HEAD ActiveRecord::Schema.define(version: 20150330083700) do -======= -ActiveRecord::Schema.define(version: 20150302130224) do ->>>>>>> Contact importing - # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -207,6 +202,7 @@ ActiveRecord::Schema.define(version: 20150302130224) do t.string "creator_str" t.string "updator_str" t.text "whois_body" + t.integer "legacy_id" end create_table "epp_sessions", force: :cascade do |t| @@ -574,6 +570,7 @@ ActiveRecord::Schema.define(version: 20150302130224) do t.integer "domain_id" t.string "creator_str" t.string "updator_str" + t.integer "legacy_id" end create_table "registrars", force: :cascade do |t| diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 5bb18b73f..5e7154df4 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,5 +1,5 @@ namespace :import do - desc 'Imports registrars' + desc 'Import registrars' task registrars: :environment do start = Time.now.to_f puts '-----> Importing registrars...' @@ -81,6 +81,7 @@ namespace :import do "city", "street", "zip", + "created_at", "street2", "street3", "creator_str", @@ -92,22 +93,20 @@ namespace :import do contacts, addresses = [], [] existing_contact_ids = Contact.pluck(:legacy_id) - existing_address_ids = Address.pluck(:legacy_contact_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 Legacy::Contact.includes(:object_registry, :object, :object_registry => :registrar).find_each(batch_size: 10000).with_index do |x, index| next if existing_contact_ids.include?(x.id) - next if existing_address_ids.include?(x.id) count += 1 begin contacts << [ - x.object_registry.name, + x.object_registry.name.try(:strip), x.telephone.try(:strip), x.email.try(:strip), x.fax.try(:strip), - x.try(:crdate), + x.object_registry.try(:crdate), x.ssn.try(:strip), ident_type_map[x.ssntype], x.object.authinfopw.try(:strip), @@ -124,6 +123,7 @@ namespace :import do x.city.try(:strip), x.street1.try(:strip), x.postalcode.try(:strip), + x.object_registry.try(:crdate), x.street2.try(:strip), x.street3.try(:strip), user, @@ -158,4 +158,73 @@ namespace :import do ) puts "-----> Imported #{count} new contacts in #{(Time.now.to_f - start).round(2)} seconds" end + + desc 'Import domains' + task domains: :environment do + start = Time.now.to_f + puts '-----> Importing domains...' + + + domain_columns = [ + "name", + "registrar_id", + "registered_at", + "status", + "valid_from", + "valid_to", + "owner_contact_id", + "auth_info", + "created_at", + "name_dirty", + "name_puny", + "period", + "period_unit", + "creator_str", + "updator_str", + "legacy_id" + ] + + domains, nameservers = [], [] + existing_domain_ids = Domain.pluck(:legacy_id) + user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" + count = 0 + + Legacy::Domain.includes(:object_registry, :object, :registrant, :object_state, :object_registry => :registrar, :object_state => :enum_object_state).find_each(batch_size: 10000).with_index do |x, index| + next if existing_domain_ids.include?(x.id) + count += 1 + + begin + domains << [ + x.object_registry.name.try(:strip), + x.object_registry.try(:registrar).try(:id), + x.object_registry.try(:crdate), + x.object_state.try(:name).try(:strip), + x.object_registry.try(:crdate), + x.exdate, + x.registrant.try(:id), + x.object.authinfopw.try(:strip), + x.object_registry.try(:crdate), + x.object_registry.name.try(:strip), + SimpleIDN.to_ascii(x.object_registry.name.try(:strip)), + 1, + 'y', + user, + user, + x.id + ] + + if domains.size % 10000 == 0 + Domain.import domain_columns, domains, validate: false + # Address.import address_columns, addresses, validate: false + domains, nameservers = [], [] + end + rescue => e + binding.pry + end + end + + Domain.import domain_columns, domains, validate: false + + puts "-----> Imported #{count} new domains in #{(Time.now.to_f - start).round(2)} seconds" + end end From 888d38486f90cddb246fcc940446d63e4f121f6c Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Tue, 3 Mar 2015 17:21:29 +0200 Subject: [PATCH 11/29] Import nameservers with domains --- app/models/legacy/domain.rb | 2 + app/models/legacy/dsrecord.rb | 5 +++ app/models/legacy/host.rb | 7 ++++ app/models/legacy/host_ipaddr_map.rb | 5 +++ app/models/legacy/keyset.rb | 7 ++++ app/models/legacy/nsset.rb | 7 ++++ ...302161712_add_legacy_columns_for_domain.rb | 2 +- db/schema.rb | 2 +- lib/tasks/import.rake | 40 ++++++++++++++++++- 9 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 app/models/legacy/dsrecord.rb create mode 100644 app/models/legacy/host.rb create mode 100644 app/models/legacy/host_ipaddr_map.rb create mode 100644 app/models/legacy/keyset.rb create mode 100644 app/models/legacy/nsset.rb diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index a28999e0e..d100fe235 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -5,6 +5,8 @@ module Legacy belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id belongs_to :object_state, foreign_key: :id, primary_key: :object_id + belongs_to :nsset, foreign_key: :nsset + belongs_to :keyset, foreign_key: :keyset belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' end end diff --git a/app/models/legacy/dsrecord.rb b/app/models/legacy/dsrecord.rb new file mode 100644 index 000000000..4630a7549 --- /dev/null +++ b/app/models/legacy/dsrecord.rb @@ -0,0 +1,5 @@ +module Legacy + class Dsrecord < Db + self.table_name = :dsrecord + end +end diff --git a/app/models/legacy/host.rb b/app/models/legacy/host.rb new file mode 100644 index 000000000..4c4eeab09 --- /dev/null +++ b/app/models/legacy/host.rb @@ -0,0 +1,7 @@ +module Legacy + class Host < Db + self.table_name = :host + + has_many :host_ipaddr_maps, foreign_key: :hostid + end +end diff --git a/app/models/legacy/host_ipaddr_map.rb b/app/models/legacy/host_ipaddr_map.rb new file mode 100644 index 000000000..a4dc2f14e --- /dev/null +++ b/app/models/legacy/host_ipaddr_map.rb @@ -0,0 +1,5 @@ +module Legacy + class HostIpaddrMap < Db + self.table_name = :host_ipaddr_map + end +end diff --git a/app/models/legacy/keyset.rb b/app/models/legacy/keyset.rb new file mode 100644 index 000000000..a0a3e1d87 --- /dev/null +++ b/app/models/legacy/keyset.rb @@ -0,0 +1,7 @@ +module Legacy + class Keyset < Db + self.table_name = :keyset + + has_many :dsrecords, foreign_key: :keysetid + end +end diff --git a/app/models/legacy/nsset.rb b/app/models/legacy/nsset.rb new file mode 100644 index 000000000..9b8a84469 --- /dev/null +++ b/app/models/legacy/nsset.rb @@ -0,0 +1,7 @@ +module Legacy + class Nsset < Db + self.table_name = :nsset + + has_many :hosts, foreign_key: :nssetid + end +end diff --git a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb index 688ae9a8e..3e3daf988 100644 --- a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb +++ b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb @@ -1,6 +1,6 @@ class AddLegacyColumnsForDomain < ActiveRecord::Migration def change add_column :domains, :legacy_id, :integer - add_column :nameservers, :legacy_id, :integer + add_column :nameservers, :legacy_domain_id, :integer end end diff --git a/db/schema.rb b/db/schema.rb index ec477ebd7..ab33f9e23 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -570,7 +570,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.integer "domain_id" t.string "creator_str" t.string "updator_str" - t.integer "legacy_id" + t.integer "legacy_domain_id" end create_table "registrars", force: :cascade do |t| diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 5e7154df4..e512f2f8b 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -184,12 +184,21 @@ namespace :import do "legacy_id" ] + nameserver_columns = [ + "hostname", + "ipv4", + "ipv6", + "creator_str", + "updator_str", + "legacy_domain_id" + ] + domains, nameservers = [], [] existing_domain_ids = Domain.pluck(:legacy_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 - Legacy::Domain.includes(:object_registry, :object, :registrant, :object_state, :object_registry => :registrar, :object_state => :enum_object_state).find_each(batch_size: 10000).with_index do |x, index| + Legacy::Domain.includes(:object_registry, :object, :registrant, :nsset, :object_state, :object_registry => :registrar, :object_state => :enum_object_state).find_each(batch_size: 10000).with_index do |x, index| next if existing_domain_ids.include?(x.id) count += 1 @@ -213,9 +222,35 @@ namespace :import do x.id ] + # nameservers + x.nsset.hosts.includes(:host_ipaddr_maps).each do |host| + ip_maps = host.host_ipaddr_maps + ips = {} + ip_maps.each do |ip_map| + next unless ip_map.ipaddr + ips[:ipv4] = ip_map.ipaddr.to_s if ip_map.ipaddr.ipv4? + ips[:ipv6] = ip_map.ipaddr.to_s if ip_map.ipaddr.ipv6? + end if ip_maps.any? + + nameservers << [ + host.fqdn.try(:strip), + ips[:ipv4].try(:strip), + ips[:ipv6].try(:strip), + user, + user, + x.id + ] + end + + # dnskeys + # x.keyset.includes(:dsrecords).each do |ds| + + # end + if domains.size % 10000 == 0 + puts 'importing' Domain.import domain_columns, domains, validate: false - # Address.import address_columns, addresses, validate: false + Nameserver.import nameserver_columns, nameservers, validate: false domains, nameservers = [], [] end rescue => e @@ -224,6 +259,7 @@ namespace :import do end Domain.import domain_columns, domains, validate: false + Nameserver.import nameserver_columns, nameservers, validate: false puts "-----> Imported #{count} new domains in #{(Time.now.to_f - start).round(2)} seconds" end From b11c6cbc7ca4c1d565d6f40bfb67d6293fa5484c Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Wed, 4 Mar 2015 13:37:20 +0200 Subject: [PATCH 12/29] Domain statuses import --- app/models/legacy/dnskey.rb | 5 ++ app/models/legacy/domain.rb | 5 +- app/models/legacy/dsrecord.rb | 5 -- app/models/legacy/keyset.rb | 7 -- app/models/legacy/object_state.rb | 28 ++++++- ...302161712_add_legacy_columns_for_domain.rb | 3 + db/schema.rb | 3 + lib/tasks/import.rake | 82 ++++++++++++++++--- 8 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 app/models/legacy/dnskey.rb delete mode 100644 app/models/legacy/dsrecord.rb delete mode 100644 app/models/legacy/keyset.rb diff --git a/app/models/legacy/dnskey.rb b/app/models/legacy/dnskey.rb new file mode 100644 index 000000000..49f8570ee --- /dev/null +++ b/app/models/legacy/dnskey.rb @@ -0,0 +1,5 @@ +module Legacy + class Dnskey < Db + self.table_name = :dnskey + end +end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index d100fe235..b443eb8f0 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -4,9 +4,10 @@ module Legacy belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id - belongs_to :object_state, foreign_key: :id, primary_key: :object_id belongs_to :nsset, foreign_key: :nsset - belongs_to :keyset, foreign_key: :keyset belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' + + has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id + has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset end end diff --git a/app/models/legacy/dsrecord.rb b/app/models/legacy/dsrecord.rb deleted file mode 100644 index 4630a7549..000000000 --- a/app/models/legacy/dsrecord.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Legacy - class Dsrecord < Db - self.table_name = :dsrecord - end -end diff --git a/app/models/legacy/keyset.rb b/app/models/legacy/keyset.rb deleted file mode 100644 index a0a3e1d87..000000000 --- a/app/models/legacy/keyset.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Legacy - class Keyset < Db - self.table_name = :keyset - - has_many :dsrecords, foreign_key: :keysetid - end -end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 39767f69c..3920cce85 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -2,8 +2,32 @@ module Legacy class ObjectState < Db self.table_name = :object_state - belongs_to :enum_object_state, foreign_key: :state_id + def name + map = { + 2 => "serverRenewProhibited", + 5 => "serverOutzoneManual", + 6 => "serverInzoneManual", + 7 => "serverBlocked", + 8 => "expirationWarning", + 9 => "expired", + 10 => "unguarded", + 11 => "validationWarning1", + 12 => "validationWarning2", + 13 => "notValidated", + 14 => "nssetMissing", + 15 => "outzone", + 18 => "serverRegistrantChangeProhibited", + 19 => "deleteWarning", + 20 => "outzoneUnguarded", + 1 => "serverDeleteProhibited", + 3 => "serverTransferProhibited", + 4 => "serverUpdateProhibited", + 16 => "linked", + 17 => "deleteCandidate", + 21 => "forceDelete" + } - delegate :name, to: :enum_object_state, prefix: false, allow_nil: true + map[state_id] + end end end diff --git a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb index 3e3daf988..aaf1f72c3 100644 --- a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb +++ b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb @@ -2,5 +2,8 @@ class AddLegacyColumnsForDomain < ActiveRecord::Migration def change add_column :domains, :legacy_id, :integer add_column :nameservers, :legacy_domain_id, :integer + add_column :dnskeys, :legacy_domain_id, :integer + add_column :domain_contacts, :legacy_domain_id, :integer + add_column :domain_statuses, :legacy_domain_id, :integer end end diff --git a/db/schema.rb b/db/schema.rb index ab33f9e23..afc260335 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "ds_digest" t.string "creator_str" t.string "updator_str" + t.integer "legacy_domain_id" end create_table "domain_contacts", force: :cascade do |t| @@ -160,6 +161,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "creator_str" t.string "updator_str" t.string "type" + t.integer "legacy_domain_id" end create_table "domain_statuses", force: :cascade do |t| @@ -168,6 +170,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "value" t.string "creator_str" t.string "updator_str" + t.integer "legacy_domain_id" end create_table "domain_transfers", force: :cascade do |t| diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index e512f2f8b..e4ebb5209 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -164,12 +164,10 @@ namespace :import do start = Time.now.to_f puts '-----> Importing domains...' - domain_columns = [ "name", "registrar_id", "registered_at", - "status", "valid_from", "valid_to", "owner_contact_id", @@ -184,6 +182,25 @@ namespace :import do "legacy_id" ] + domain_contact_columns = [ + "contact_id", + "domain_id", + "contact_type", + "created_at", + "updated_at", + "contact_code_cache", + "creator_str", + "updator_str", + "legacy_domain_id" + ] + + domain_status_columns = [ + "value", + "creator_str", + "updator_str", + "legacy_domain_id" + ] + nameserver_columns = [ "hostname", "ipv4", @@ -193,12 +210,26 @@ namespace :import do "legacy_domain_id" ] - domains, nameservers = [], [] + dnskey_columns = [ + "flags", + "protocol", + "alg", + "public_key", + "ds_alg", + "ds_digest_type", + "creator_str", + "updator_str", + "legacy_domain_id" + ] + + + + domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] existing_domain_ids = Domain.pluck(:legacy_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 - Legacy::Domain.includes(:object_registry, :object, :registrant, :nsset, :object_state, :object_registry => :registrar, :object_state => :enum_object_state).find_each(batch_size: 10000).with_index do |x, index| + Legacy::Domain.includes(:object_registry, :object, :registrant, :nsset, :object_states, :dnskeys, :object_registry => :registrar).find_each(batch_size: 10000).with_index do |x, index| next if existing_domain_ids.include?(x.id) count += 1 @@ -207,7 +238,6 @@ namespace :import do x.object_registry.name.try(:strip), x.object_registry.try(:registrar).try(:id), x.object_registry.try(:crdate), - x.object_state.try(:name).try(:strip), x.object_registry.try(:crdate), x.exdate, x.registrant.try(:id), @@ -222,6 +252,19 @@ namespace :import do x.id ] + # domain contacts + + + # domain statuses + x.object_states.each do |state| + domain_statuses << [ + state.name.try(:strip), + user, + user, + x.id + ] + end + # nameservers x.nsset.hosts.includes(:host_ipaddr_maps).each do |host| ip_maps = host.host_ipaddr_maps @@ -242,16 +285,27 @@ namespace :import do ] end - # dnskeys - # x.keyset.includes(:dsrecords).each do |ds| + x.dnskeys.each do |key| + dnskeys << [ + key.flags, + key.protocol, + key.alg, + key.key, + 3, # ds_alg + Setting.ds_algorithm, # ds_digest_type + user, + user, + x.id + ] + end - # end - - if domains.size % 10000 == 0 + if index % 10000 == 0 && index != 0 puts 'importing' Domain.import domain_columns, domains, validate: false Nameserver.import nameserver_columns, nameservers, validate: false - domains, nameservers = [], [] + Dnskey.import dnskey_columns, dnskeys, validate: false + DomainStatus.import domain_status_columns, domain_statuses, validate: false + domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] end rescue => e binding.pry @@ -260,6 +314,12 @@ namespace :import do Domain.import domain_columns, domains, validate: false Nameserver.import nameserver_columns, nameservers, validate: false + Dnskey.import dnskey_columns, dnskeys, validate: false + DomainStatus.import domain_status_columns, domain_statuses, validate: false + + # puts '-----> Updating relations...' + + # puts '-----> Generating dnskey digests...' puts "-----> Imported #{count} new domains in #{(Time.now.to_f - start).round(2)} seconds" end From 0699c38584dc99dddb9b419ac0fcb858651c21fa Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Wed, 4 Mar 2015 17:32:54 +0200 Subject: [PATCH 13/29] Import domain contacts --- app/models/legacy/domain.rb | 3 +- app/models/legacy/domain_contact_map.rb | 7 +++ app/models/legacy/object_state.rb | 28 ++++++++++ ...302161712_add_legacy_columns_for_domain.rb | 3 ++ db/schema.rb | 5 +- lib/tasks/import.rake | 51 ++++++++++++------- 6 files changed, 76 insertions(+), 21 deletions(-) create mode 100644 app/models/legacy/domain_contact_map.rb diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index b443eb8f0..ffb72657b 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -5,9 +5,10 @@ module Legacy belongs_to :object_registry, foreign_key: :id belongs_to :object, foreign_key: :id belongs_to :nsset, foreign_key: :nsset - belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' + # belongs_to :registrant, foreign_key: :registrant, primary_key: :legacy_id, class_name: '::Contact' has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset + has_many :domain_contact_maps, foreign_key: :domainid end end diff --git a/app/models/legacy/domain_contact_map.rb b/app/models/legacy/domain_contact_map.rb new file mode 100644 index 000000000..ff7806389 --- /dev/null +++ b/app/models/legacy/domain_contact_map.rb @@ -0,0 +1,7 @@ +module Legacy + class DomainContactMap < Db + self.table_name = :domain_contact_map + + belongs_to :contact, foreign_key: :contactid, primary_key: :legacy_id, class_name: '::Contact' + end +end diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index 3920cce85..f40f7a27d 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -29,5 +29,33 @@ module Legacy map[state_id] end + + def desc + map = { + 1 => "Delete prohibited", + 2 => "Registration renew prohibited", + 3 => "Sponsoring registrar change prohibited", + 4 => "Update prohibited", + 7 => "Domain blocked", + 8 => "Expires within 30 days", + 9 => "Expired", + 10 => "Domain is 30 days after expiration", + 11 => "Validation of domain expire in 30 days", + 12 => "Validation of domain expire in 15 days", + 13 => "Domain not validated", + 14 => "Domain has not associated nsset", + 15 => "Domain is not generated into zone", + 16 => "Has relation to other records in registry", + 17 => "Object is going to be deleted", + 18 => "Registrant change prohibited", + 19 => "Domain will be deleted in 11 days", + 20 => "Domain is out of zone after 30 days from expiration", + 21 => "Domain is forced to delete", + 5 => "Domain is administratively kept out of zone", + 6 => "Domain is administratively kept in zone" + } + + map[state_id] + end end end diff --git a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb index aaf1f72c3..cd6792ce4 100644 --- a/db/migrate/20150302161712_add_legacy_columns_for_domain.rb +++ b/db/migrate/20150302161712_add_legacy_columns_for_domain.rb @@ -1,9 +1,12 @@ class AddLegacyColumnsForDomain < ActiveRecord::Migration def change add_column :domains, :legacy_id, :integer + add_column :domains, :legacy_registrar_id, :integer + add_column :domains, :legacy_registrant_id, :integer add_column :nameservers, :legacy_domain_id, :integer add_column :dnskeys, :legacy_domain_id, :integer add_column :domain_contacts, :legacy_domain_id, :integer + add_column :domain_contacts, :legacy_contact_id, :integer add_column :domain_statuses, :legacy_domain_id, :integer end end diff --git a/db/schema.rb b/db/schema.rb index afc260335..88a0dfe2e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -162,6 +162,7 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "updator_str" t.string "type" t.integer "legacy_domain_id" + t.integer "legacy_contact_id" end create_table "domain_statuses", force: :cascade do |t| @@ -201,11 +202,13 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "name_dirty" t.string "name_puny" t.integer "period" - t.string "period_unit", limit: 1 + t.string "period_unit", limit: 1 t.string "creator_str" t.string "updator_str" t.text "whois_body" t.integer "legacy_id" + t.integer "legacy_registrar_id" + t.integer "legacy_registrant_id" end create_table "epp_sessions", force: :cascade do |t| diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index e4ebb5209..d4a8545c4 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -166,11 +166,9 @@ namespace :import do domain_columns = [ "name", - "registrar_id", "registered_at", "valid_from", "valid_to", - "owner_contact_id", "auth_info", "created_at", "name_dirty", @@ -179,22 +177,21 @@ namespace :import do "period_unit", "creator_str", "updator_str", - "legacy_id" + "legacy_id", + "legacy_registrar_id", + "legacy_registrant_id" ] domain_contact_columns = [ - "contact_id", - "domain_id", "contact_type", - "created_at", - "updated_at", - "contact_code_cache", "creator_str", "updator_str", - "legacy_domain_id" + "legacy_domain_id", + "legacy_contact_id" ] domain_status_columns = [ + "description", "value", "creator_str", "updator_str", @@ -222,25 +219,29 @@ namespace :import do "legacy_domain_id" ] - - domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] existing_domain_ids = Domain.pluck(:legacy_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 - Legacy::Domain.includes(:object_registry, :object, :registrant, :nsset, :object_states, :dnskeys, :object_registry => :registrar).find_each(batch_size: 10000).with_index do |x, index| + Legacy::Domain.includes( + :object_registry, + :object, + :nsset, + :object_states, + :dnskeys, + :domain_contact_maps, + :nsset => {:hosts => :host_ipaddr_maps} + ).find_each(batch_size: 10000).with_index do |x, index| next if existing_domain_ids.include?(x.id) count += 1 begin domains << [ x.object_registry.name.try(:strip), - x.object_registry.try(:registrar).try(:id), x.object_registry.try(:crdate), x.object_registry.try(:crdate), x.exdate, - x.registrant.try(:id), x.object.authinfopw.try(:strip), x.object_registry.try(:crdate), x.object_registry.name.try(:strip), @@ -249,16 +250,27 @@ namespace :import do 'y', user, user, - x.id + x.id, + x.object_registry.try(:crid), + x.registrant ] # domain contacts - + x.domain_contact_maps.each do |dc| + domain_contacts << [ + 'admin', # TODO: Where to get real contact type? + user, + user, + x.id, + dc.contactid + ] + end # domain statuses x.object_states.each do |state| domain_statuses << [ - state.name.try(:strip), + state.desc, + state.name, user, user, x.id @@ -266,7 +278,7 @@ namespace :import do end # nameservers - x.nsset.hosts.includes(:host_ipaddr_maps).each do |host| + x.nsset.hosts.each do |host| ip_maps = host.host_ipaddr_maps ips = {} ip_maps.each do |ip_map| @@ -300,11 +312,11 @@ namespace :import do end if index % 10000 == 0 && index != 0 - puts 'importing' Domain.import domain_columns, domains, validate: false Nameserver.import nameserver_columns, nameservers, validate: false Dnskey.import dnskey_columns, dnskeys, validate: false DomainStatus.import domain_status_columns, domain_statuses, validate: false + DomainContact.import domain_contact_columns, domain_contacts, validate: false domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] end rescue => e @@ -316,6 +328,7 @@ namespace :import do Nameserver.import nameserver_columns, nameservers, validate: false Dnskey.import dnskey_columns, dnskeys, validate: false DomainStatus.import domain_status_columns, domain_statuses, validate: false + DomainContact.import domain_contact_columns, domain_contacts, validate: false # puts '-----> Updating relations...' From 1e76dfa35df821f51c6b746af183d8329b858aaf Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Wed, 4 Mar 2015 17:56:55 +0200 Subject: [PATCH 14/29] Relations update --- lib/tasks/import.rake | 79 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index d4a8545c4..5ddbd5823 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -330,9 +330,84 @@ namespace :import do DomainStatus.import domain_status_columns, domain_statuses, validate: false DomainContact.import domain_contact_columns, domain_contacts, validate: false - # puts '-----> Updating relations...' + puts '-----> Updating relations...' - # puts '-----> Generating dnskey digests...' + # registrant + ActiveRecord::Base.connection.execute( + "UPDATE domains "\ + "SET owner_contact_id = contacts.id "\ + "FROM contacts "\ + "WHERE contacts.legacy_id = legacy_registrant_id "\ + "AND legacy_registrant_id IS NOT NULL "\ + "AND owner_contact_id IS NULL" + ) + + # registrar + ActiveRecord::Base.connection.execute( + "UPDATE domains "\ + "SET registrar_id = registrars.id "\ + "FROM registrars "\ + "WHERE registrars.legacy_id = legacy_registrar_id "\ + "AND legacy_registrar_id IS NOT NULL "\ + "AND registrar_id IS NULL" + ) + + # contacts + ActiveRecord::Base.connection.execute( + "UPDATE domain_contacts "\ + "SET contact_id = contacts.id "\ + "FROM contacts "\ + "WHERE contacts.legacy_id = legacy_contact_id "\ + "AND legacy_contact_id IS NOT NULL "\ + "AND contact_id IS NULL" + ) + + ActiveRecord::Base.connection.execute( + "UPDATE domain_contacts "\ + "SET domain_id = domains.id "\ + "FROM domains "\ + "WHERE domains.legacy_id = legacy_domain_id "\ + "AND legacy_domain_id IS NOT NULL "\ + "AND domain_id IS NULL" + ) + + # nameservers + ActiveRecord::Base.connection.execute( + "UPDATE nameservers "\ + "SET domain_id = domains.id "\ + "FROM domains "\ + "WHERE domains.legacy_id = legacy_domain_id "\ + "AND legacy_domain_id IS NOT NULL "\ + "AND domain_id IS NULL" + ) + + # dnskeys + ActiveRecord::Base.connection.execute( + "UPDATE dnskeys "\ + "SET domain_id = domains.id "\ + "FROM domains "\ + "WHERE domains.legacy_id = legacy_domain_id "\ + "AND legacy_domain_id IS NOT NULL "\ + "AND domain_id IS NULL" + ) + + # statuses + ActiveRecord::Base.connection.execute( + "UPDATE domain_statuses "\ + "SET domain_id = domains.id "\ + "FROM domains "\ + "WHERE domains.legacy_id = legacy_domain_id "\ + "AND legacy_domain_id IS NOT NULL "\ + "AND domain_id IS NULL" + ) + + puts '-----> Generating dnskey digests...' + + Dnskey.all.each do |x| + x.generate_digest + x.generate_ds_key_tag + x.save(validate: false) + end puts "-----> Imported #{count} new domains in #{(Time.now.to_f - start).round(2)} seconds" end From d5a7704b8746756f898d0f4b747ad3cad6f76995 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Wed, 4 Mar 2015 18:06:30 +0200 Subject: [PATCH 15/29] Remove binding.pry --- lib/tasks/import.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 5ddbd5823..419ae4987 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -320,7 +320,8 @@ namespace :import do domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] end rescue => e - binding.pry + puts "ERROR on index #{index}" + puts e end end From 968e383f9e2c5dc6ff1b28e3fa6f21dd30342405 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Tue, 17 Mar 2015 11:50:45 +0200 Subject: [PATCH 16/29] Import tech contacts too --- app/models/contact.rb | 2 +- app/models/dnskey.rb | 4 ++-- app/models/legacy/domain.rb | 1 + app/models/legacy/domain_contact_map.rb | 2 +- app/models/legacy/nsset_contact_map.rb | 5 +++++ lib/tasks/import.rake | 17 +++++++++++++++-- 6 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 app/models/legacy/nsset_contact_map.rb diff --git a/app/models/contact.rb b/app/models/contact.rb index 97ae8e1ca..79a0e29c0 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -84,7 +84,7 @@ class Contact < ActiveRecord::Base end def to_s - name + name || '[no name]' end def ident_valid_format? diff --git a/app/models/dnskey.rb b/app/models/dnskey.rb index fbdeb80ec..e3b4c8963 100644 --- a/app/models/dnskey.rb +++ b/app/models/dnskey.rb @@ -73,9 +73,9 @@ class Dnskey < ActiveRecord::Base hex = [domain.name_in_wire_format, flags_hex, protocol_hex, alg_hex, public_key_hex].join bin = self.class.hex_to_bin(hex) - if Setting.ds_algorithm == 1 + if ds_digest_type == 1 self.ds_digest = Digest::SHA1.hexdigest(bin).upcase - elsif Setting.ds_algorithm == 2 + elsif ds_digest_type == 2 self.ds_digest = Digest::SHA256.hexdigest(bin).upcase end end diff --git a/app/models/legacy/domain.rb b/app/models/legacy/domain.rb index ffb72657b..5f9568170 100644 --- a/app/models/legacy/domain.rb +++ b/app/models/legacy/domain.rb @@ -10,5 +10,6 @@ module Legacy has_many :object_states, -> { where('valid_to IS NULL') }, foreign_key: :object_id has_many :dnskeys, foreign_key: :keysetid, primary_key: :keyset has_many :domain_contact_maps, foreign_key: :domainid + has_many :nsset_contact_maps, foreign_key: :nssetid, primary_key: :nsset end end diff --git a/app/models/legacy/domain_contact_map.rb b/app/models/legacy/domain_contact_map.rb index ff7806389..4e645ed7f 100644 --- a/app/models/legacy/domain_contact_map.rb +++ b/app/models/legacy/domain_contact_map.rb @@ -2,6 +2,6 @@ module Legacy class DomainContactMap < Db self.table_name = :domain_contact_map - belongs_to :contact, foreign_key: :contactid, primary_key: :legacy_id, class_name: '::Contact' + # belongs_to :contact, foreign_key: :contactid, primary_key: :legacy_id, class_name: '::Contact' end end diff --git a/app/models/legacy/nsset_contact_map.rb b/app/models/legacy/nsset_contact_map.rb new file mode 100644 index 000000000..f7e0a105b --- /dev/null +++ b/app/models/legacy/nsset_contact_map.rb @@ -0,0 +1,5 @@ +module Legacy + class NssetContactMap < Db + self.table_name = :nsset_contact_map + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 419ae4987..eb9e13b29 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -164,6 +164,8 @@ namespace :import do start = Time.now.to_f puts '-----> Importing domains...' + # domain_status_map = {} + domain_columns = [ "name", "registered_at", @@ -255,7 +257,7 @@ namespace :import do x.registrant ] - # domain contacts + # admin contacts x.domain_contact_maps.each do |dc| domain_contacts << [ 'admin', # TODO: Where to get real contact type? @@ -266,6 +268,17 @@ namespace :import do ] end + # tech contacts + x.nsset_contact_maps.each do |dc| + domain_contacts << [ + 'tech', # TODO: Where to get real contact type? + user, + user, + x.id, + dc.contactid + ] + end + # domain statuses x.object_states.each do |state| domain_statuses << [ @@ -304,7 +317,7 @@ namespace :import do key.alg, key.key, 3, # ds_alg - Setting.ds_algorithm, # ds_digest_type + 1, # ds_digest_type /SHA1) user, user, x.id From 3a525b6a111080799c49a9259359e2aca7961b50 Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Thu, 19 Mar 2015 09:49:15 +0200 Subject: [PATCH 17/29] Ignore old statuses --- app/models/legacy/object_state.rb | 39 +++++++++++++++++++++++-------- lib/tasks/import.rake | 1 + 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/models/legacy/object_state.rb b/app/models/legacy/object_state.rb index f40f7a27d..0a59cf021 100644 --- a/app/models/legacy/object_state.rb +++ b/app/models/legacy/object_state.rb @@ -3,28 +3,47 @@ module Legacy self.table_name = :object_state def name + # legacy values + # 2 => "serverRenewProhibited", + # 5 => "serverOutzoneManual", + # 6 => "serverInzoneManual", + # 7 => "serverBlocked", + # 8 => "expirationWarning", + # 9 => "expired", + # 10 => "unguarded", + # 11 => "validationWarning1", + # 12 => "validationWarning2", + # 13 => "notValidated", + # 14 => "nssetMissing", + # 15 => "outzone", + # 18 => "serverRegistrantChangeProhibited", + # 19 => "deleteWarning", + # 20 => "outzoneUnguarded", + # 1 => "serverDeleteProhibited", + # 3 => "serverTransferProhibited", + # 4 => "serverUpdateProhibited", + # 16 => "linked", + # 17 => "deleteCandidate", + # 21 => "forceDelete" + + # new values map = { 2 => "serverRenewProhibited", - 5 => "serverOutzoneManual", - 6 => "serverInzoneManual", + 5 => "serverHold", + 6 => "serverManualInzone", 7 => "serverBlocked", - 8 => "expirationWarning", 9 => "expired", - 10 => "unguarded", 11 => "validationWarning1", - 12 => "validationWarning2", 13 => "notValidated", 14 => "nssetMissing", - 15 => "outzone", + 15 => "serverHold", 18 => "serverRegistrantChangeProhibited", - 19 => "deleteWarning", - 20 => "outzoneUnguarded", 1 => "serverDeleteProhibited", 3 => "serverTransferProhibited", 4 => "serverUpdateProhibited", 16 => "linked", - 17 => "deleteCandidate", - 21 => "forceDelete" + 17 => "deleteCandidate", # grupistaatus + 21 => "forceDelete" # grupistaatus } map[state_id] diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index eb9e13b29..8ab52d610 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -281,6 +281,7 @@ namespace :import do # domain statuses x.object_states.each do |state| + next if state.name.blank? domain_statuses << [ state.desc, state.name, From ad1749c1de1a0b97ffa6dc9032b73c68fbf956be Mon Sep 17 00:00:00 2001 From: Martin Lensment Date: Tue, 31 Mar 2015 11:20:55 +0300 Subject: [PATCH 18/29] Remove todo-s --- lib/tasks/import.rake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 8ab52d610..3603937b4 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -164,8 +164,6 @@ namespace :import do start = Time.now.to_f puts '-----> Importing domains...' - # domain_status_map = {} - domain_columns = [ "name", "registered_at", @@ -260,7 +258,7 @@ namespace :import do # admin contacts x.domain_contact_maps.each do |dc| domain_contacts << [ - 'admin', # TODO: Where to get real contact type? + 'admin', user, user, x.id, @@ -271,7 +269,7 @@ namespace :import do # tech contacts x.nsset_contact_maps.each do |dc| domain_contacts << [ - 'tech', # TODO: Where to get real contact type? + 'tech', user, user, x.id, From 29f1aa9d0cd89e713020c14a57a76bee51ec4bcd Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:19:00 +0300 Subject: [PATCH 19/29] Upgraded hashie_rails gem, it issued depricated warning --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 85f348462..dcdae4548 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' # core gem 'rails', '4.2.1' gem 'iso8601', '~> 0.8.2' # for dates and times -gem 'hashie_rails', '~> 0.0.1' +gem 'hashie-forbidden_attributes', '~> 0.1.1' # load env gem 'figaro', '~> 1.1.0' diff --git a/Gemfile.lock b/Gemfile.lock index f92ac9739..8a237dad4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -240,7 +240,7 @@ GEM html2haml (>= 1.0.1) railties (>= 4.0.1) hashie (3.4.1) - hashie_rails (0.0.4) + hashie-forbidden_attributes (0.1.1) hashie (>= 3.0) highline (1.6.21) hike (1.2.3) @@ -537,7 +537,7 @@ DEPENDENCIES guard-rspec (~> 4.3.1) guard-rubocop (~> 1.1.0) haml-rails (~> 0.9.0) - hashie_rails (~> 0.0.1) + hashie-forbidden_attributes (~> 0.1.1) html2haml! isikukood iso8601 (~> 0.8.2) From 3d78dc97cd4abcd3c5f2b1e572f3c832d9097b59 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:20:00 +0300 Subject: [PATCH 20/29] Removed depricated address migration --- db/migrate/20150302130224_add_legacy_columns_for_address.rb | 5 ----- db/schema.rb | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 db/migrate/20150302130224_add_legacy_columns_for_address.rb diff --git a/db/migrate/20150302130224_add_legacy_columns_for_address.rb b/db/migrate/20150302130224_add_legacy_columns_for_address.rb deleted file mode 100644 index 3a2d4e8ba..000000000 --- a/db/migrate/20150302130224_add_legacy_columns_for_address.rb +++ /dev/null @@ -1,5 +0,0 @@ -class AddLegacyColumnsForAddress < ActiveRecord::Migration - def change - add_column :addresses, :legacy_contact_id, :integer - end -end diff --git a/db/schema.rb b/db/schema.rb index 88a0dfe2e..464f2dd8c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -12,6 +12,7 @@ # It's strongly recommended that you check this file into your version control system. ActiveRecord::Schema.define(version: 20150330083700) do + # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" From c239460d6c8d295d9a50b0ed826063771e0cf207 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:37:08 +0300 Subject: [PATCH 21/29] import syntax updates for rubocop --- lib/tasks/import.rake | 166 +++++++++++++++++++++--------------------- 1 file changed, 84 insertions(+), 82 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 3603937b4..2b9e71cdf 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -59,44 +59,46 @@ namespace :import do 6 => Contact::IDENT_BIRTHDAY } - contact_columns = [ - "code", - "phone", - "email", - "fax", - "created_at", - "ident", - "ident_type", - "auth_info", - "name", - "org_name", - "registrar_id", - "creator_str", - "updator_str", - "ident_country_code", - "legacy_id" - ] + contact_columns = %w( + code + phone + email + fax + created_at + ident + ident_type + auth_info + name + org_name + registrar_id + creator_str + updator_str + ident_country_code + legacy_id + ) - address_columns = [ - "city", - "street", - "zip", - "created_at", - "street2", - "street3", - "creator_str", - "updator_str", - "country_code", - "state", - "legacy_contact_id" - ] + address_columns = %w( + city + street + zip + created_at + street2 + street3 + creator_str + updator_str + country_code + state + legacy_contact_id + ) contacts, addresses = [], [] existing_contact_ids = Contact.pluck(:legacy_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 - Legacy::Contact.includes(:object_registry, :object, :object_registry => :registrar).find_each(batch_size: 10000).with_index do |x, index| + Legacy::Contact.includes(:object_registry, :object, object_registry: :registrar) + .find_each(batch_size: 10000).with_index do |x, index| + next if existing_contact_ids.include?(x.id) count += 1 @@ -164,60 +166,60 @@ namespace :import do start = Time.now.to_f puts '-----> Importing domains...' - domain_columns = [ - "name", - "registered_at", - "valid_from", - "valid_to", - "auth_info", - "created_at", - "name_dirty", - "name_puny", - "period", - "period_unit", - "creator_str", - "updator_str", - "legacy_id", - "legacy_registrar_id", - "legacy_registrant_id" - ] + domain_columns = %w( + name + registered_at + valid_from + valid_to + auth_info + created_at + name_dirty + name_puny + period + period_unit + creator_str + updator_str + legacy_id + legacy_registrar_id + legacy_registrant_id + ) - domain_contact_columns = [ - "contact_type", - "creator_str", - "updator_str", - "legacy_domain_id", - "legacy_contact_id" - ] + domain_contact_columns = %w( + contact_type + creator_str + updator_str + legacy_domain_id + legacy_contact_id + ) - domain_status_columns = [ - "description", - "value", - "creator_str", - "updator_str", - "legacy_domain_id" - ] + domain_status_columns = %w( + description + value + creator_str + updator_str + legacy_domain_id + ) - nameserver_columns = [ - "hostname", - "ipv4", - "ipv6", - "creator_str", - "updator_str", - "legacy_domain_id" - ] + nameserver_columns = %w( + hostname + ipv4 + ipv6 + creator_str + updator_str + legacy_domain_id + ) - dnskey_columns = [ - "flags", - "protocol", - "alg", - "public_key", - "ds_alg", - "ds_digest_type", - "creator_str", - "updator_str", - "legacy_domain_id" - ] + dnskey_columns = %w( + flags + protocol + alg + public_key + ds_alg + ds_digest_type + creator_str + updator_str + legacy_domain_id + ) domains, nameservers, dnskeys, domain_statuses, domain_contacts = [], [], [], [], [] existing_domain_ids = Domain.pluck(:legacy_id) @@ -231,7 +233,7 @@ namespace :import do :object_states, :dnskeys, :domain_contact_maps, - :nsset => {:hosts => :host_ipaddr_maps} + nsset: { hosts: :host_ipaddr_maps } ).find_each(batch_size: 10000).with_index do |x, index| next if existing_domain_ids.include?(x.id) count += 1 From a393bf04cb157ee2751473882aca4ed83f0f0ca3 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:37:59 +0300 Subject: [PATCH 22/29] Whois body generated on live --- app/models/domain.rb | 4 +--- spec/models/domain_spec.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/models/domain.rb b/app/models/domain.rb index 8222cea76..b705dacd8 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -236,7 +236,7 @@ class Domain < ActiveRecord::Base # rubocop:disable Metrics/MethodLength def update_whois_body - new_whois_body = <<-EOS + self.whois_body = <<-EOS This Whois Server contains information on Estonian Top Level Domain ee TLD @@ -261,8 +261,6 @@ class Domain < ActiveRecord::Base changed: #{registrar.updated_at.to_s(:db)} EOS - return if whois_body == new_whois_body - update_column(whois_body: new_whois_body) update_whois_server end # handle_asynchronously :update_whois_body diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index 11d33b964..3d672df95 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -75,7 +75,7 @@ describe Domain do end it 'should not have whois_body present by default' do - @domain.whois_body.present?.should == false + @domain.whois_body.present?.should == true end context 'with versioning' do From 17d130855cf9c8855021cefe8fcebff5ac01516f Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:38:21 +0300 Subject: [PATCH 23/29] removed async --- app/models/domain.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/domain.rb b/app/models/domain.rb index b705dacd8..7c81736be 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -263,7 +263,6 @@ class Domain < ActiveRecord::Base update_whois_server end - # handle_asynchronously :update_whois_body # rubocop:enable Metrics/MethodLength def contacts_body From 7d94304d560ea435b3c3eb3c59e54d7f39211f79 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 14:47:52 +0300 Subject: [PATCH 24/29] Depricated delayed jobs --- CHANGELOG.md | 4 +++ Gemfile | 2 -- Gemfile.lock | 8 ----- app/models/ability.rb | 2 +- .../concerns/domain_version_observer.rb | 33 ------------------- app/views/layouts/application.haml | 2 +- config/application.rb | 2 -- config/deploy.rb | 24 -------------- db/migrate/20150402114712_drop_delayed_job.rb | 5 +++ db/schema.rb | 18 +--------- 10 files changed, 12 insertions(+), 88 deletions(-) create mode 100644 db/migrate/20150402114712_drop_delayed_job.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index b2fbfdef5..c78017d2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +02.04.2015 + +* Depricated DelayedJob, kill all running delayed jobs if needed + 27.03.2015 * Integrated DEPP engine to Registrar. Please note new DEPP configuration in application-example.yml diff --git a/Gemfile b/Gemfile index dcdae4548..c62df99f5 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,6 @@ gem 'paper_trail', github: 'airblade/paper_trail', ref: 'a453811226ec4ea59753ba6b827e390ced2fc140' # '~> 4.0.0.beta2' # archiving gem 'rails-settings-cached', '~> 0.4.1' # for settings -gem 'delayed_job_active_record', '~> 4.0.3' # delayed job # html-xml gem 'haml-rails', '~> 0.9.0' # haml for views @@ -52,7 +51,6 @@ gem 'isikukood' # for EE-id validation # deploy gem 'whenever', '~> 0.9.4', require: false -gem 'daemons', '~> 1.1.9' # process delayed jobs # monitors gem 'newrelic_rpm', '~> 3.9.9.275' diff --git a/Gemfile.lock b/Gemfile.lock index 8a237dad4..985b84444 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -156,16 +156,10 @@ GEM crack (0.4.2) safe_yaml (~> 1.0.0) currencies (0.4.2) - daemons (1.1.9) database_cleaner (1.3.0) debug_inspector (0.0.2) deep_cloneable (2.1.1) activerecord (>= 3.1.0, < 5.0.0) - delayed_job (4.0.6) - activesupport (>= 3.0, < 5.0) - delayed_job_active_record (4.0.3) - activerecord (>= 3.0, < 5.0) - delayed_job (>= 3.0, < 4.1) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) devise (3.4.1) @@ -519,10 +513,8 @@ DEPENDENCIES capybara (~> 2.4.1) coffee-rails (~> 4.1.0) countries (~> 0.10.0) - daemons (~> 1.1.9) database_cleaner (~> 1.3.0) deep_cloneable (~> 2.1.1) - delayed_job_active_record (~> 4.0.3) depp! devise (~> 3.4.1) digidoc_client (~> 0.2.1) diff --git a/app/models/ability.rb b/app/models/ability.rb index 9098cb4e2..f351b52c4 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -71,7 +71,7 @@ class Ability can :manage, LegalDocument can :read, ApiLog::EppLog can :read, ApiLog::ReppLog - can :index, :delayed_job + # can :index, :delayed_job can :create, :zonefile can :access, :settings_menu end diff --git a/app/models/concerns/domain_version_observer.rb b/app/models/concerns/domain_version_observer.rb index 2846b48ce..f1e732f21 100644 --- a/app/models/concerns/domain_version_observer.rb +++ b/app/models/concerns/domain_version_observer.rb @@ -1,36 +1,3 @@ module DomainVersionObserver extend ActiveSupport::Concern - - # TODO: remove old - # included do - # after_save :delayed_whois_update - # end - - # private - - # def delayed_whois_update - # name = domain_name - # return unless name - # body = snapshot - # delay.update_private_whois(name, body) - # delay.update_public_whois(name, body) - # end - - # def update_private_whois(domain_name, body) - # wd = Whois::PublicDomain.find_or_initialize_by(name: domain_name) - # wd.body = body - # wd.save! - # end - - # def update_public_whois(domain_name, body) - # wd = Whois::PrivateDomain.find_or_initialize_by(name: domain_name) - # wd.body = body - # wd.save! - # end - - # def domain_name - # name = reify.try(:name) - # name = load_snapshot[:domain][:name] if event == 'create' - # return name if name - # end end diff --git a/app/views/layouts/application.haml b/app/views/layouts/application.haml index 48eed2494..68629ec2c 100644 --- a/app/views/layouts/application.haml +++ b/app/views/layouts/application.haml @@ -48,7 +48,7 @@ -# %li= link_to t(:domains_history), admin_domain_versions_path %li= link_to t(:epp_logs), admin_epp_logs_path %li= link_to t(:repp_logs), admin_repp_logs_path - %li= link_to t(:background_jobs), admin_delayed_jobs_path + -# %li= link_to t(:background_jobs), admin_delayed_jobs_path %li.divider %li.dropdown-header= t('users') diff --git a/config/application.rb b/config/application.rb index 87fdf723d..f44bb80f0 100644 --- a/config/application.rb +++ b/config/application.rb @@ -45,8 +45,6 @@ module Registry # Instead, the errors will propagate normally just like in other Active Record callbacks. config.active_record.raise_in_transactional_callbacks = true - config.active_job.queue_adapter = :delayed_job - config.generators do |g| g.stylesheets false g.javascripts false diff --git a/config/deploy.rb b/config/deploy.rb index 275f1c099..5bce5727e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -17,7 +17,6 @@ set :deploy_to, '$HOME/registry' set :repository, 'https://github.com/domify/registry' # dev repo set :branch, 'master' set :rails_env, 'alpha' -set :delayed_job, true # alpha branch task :epp do @@ -26,7 +25,6 @@ task :epp do set :repository, 'https://github.com/domify/registry' # dev repo set :branch, 'master' set :rails_env, 'alpha' - set :delayed_job, false end task :registrar do @@ -35,7 +33,6 @@ task :registrar do set :repository, 'https://github.com/domify/registry' # dev repo set :branch, 'master' set :rails_env, 'alpha' - set :delayed_job, false end # staging @@ -45,7 +42,6 @@ task :st do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' # same as production set :rails_env, 'staging' - set :delayed_job, true end # staging @@ -55,7 +51,6 @@ task :eppst do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' # same as production set :rails_env, 'staging' - set :delayed_job, false end # staging @@ -65,7 +60,6 @@ task :registrarst do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' set :rails_env, 'staging' - set :delayed_job, false end # production @@ -75,7 +69,6 @@ task :pr do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' # same as staging set :rails_env, 'production' - set :delayed_job, true end # production @@ -85,7 +78,6 @@ task :epppr do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' # same as staging set :rails_env, 'production' - set :delayed_job, false end # production @@ -95,7 +87,6 @@ task :registrarst do set :repository, 'https://github.com/internetee/registry' # production repo set :branch, 'master' set :rails_env, 'production' - set :delayed_job, false end # Manually create these paths in shared/ (eg: shared/config/database.yml) in your server. @@ -159,10 +150,6 @@ task deploy: :environment do deploy do # Put things that will set up an empty directory into a fully set-up # instance of your project. - to :prepare do - invoke :'delayed_job:stop' if delayed_job - end - invoke :'git:clone' invoke :load_commit_hash invoke :'deploy:link_shared_paths' @@ -171,7 +158,6 @@ task deploy: :environment do invoke :'rails:assets_precompile' to :launch do invoke :restart - invoke :'delayed_job:start' if delayed_job end end end @@ -197,16 +183,6 @@ task load_commit_hash: :environment do ) end -namespace :delayed_job do - task stop: :environment do - queue %(echo "-----> Stopping delayed job"; cd #{deploy_to}/current; RAILS_ENV=#{rails_env} bin/delayed_job stop) - end - - task start: :environment do - queue %(echo "-----> Starting delayed job"; cd #{deploy_to}/current; RAILS_ENV=#{rails_env} bin/delayed_job start) - end -end - desc 'Restart Passenger application' task restart: :environment do queue "mkdir -p #{deploy_to}/current/tmp; touch #{deploy_to}/current/tmp/restart.txt" diff --git a/db/migrate/20150402114712_drop_delayed_job.rb b/db/migrate/20150402114712_drop_delayed_job.rb new file mode 100644 index 000000000..b5844b76f --- /dev/null +++ b/db/migrate/20150402114712_drop_delayed_job.rb @@ -0,0 +1,5 @@ +class DropDelayedJob < ActiveRecord::Migration + def change + drop_table :delayed_jobs + end +end diff --git a/db/schema.rb b/db/schema.rb index 464f2dd8c..4f718e907 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150330083700) do +ActiveRecord::Schema.define(version: 20150402114712) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -108,22 +108,6 @@ ActiveRecord::Schema.define(version: 20150330083700) do t.string "updator_str" end - create_table "delayed_jobs", force: :cascade do |t| - t.integer "priority", default: 0, null: false - t.integer "attempts", default: 0, null: false - t.text "handler", null: false - t.text "last_error" - t.datetime "run_at" - t.datetime "locked_at" - t.datetime "failed_at" - t.string "locked_by" - t.string "queue" - t.datetime "created_at" - t.datetime "updated_at" - end - - add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree - create_table "delegation_signers", force: :cascade do |t| t.integer "domain_id" t.string "key_tag" From 393a4f6d06f8b403fe621fa4d2785dce74bdb3ab Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 17:22:32 +0300 Subject: [PATCH 25/29] Added password type --- app/models/contact.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/contact.rb b/app/models/contact.rb index 79a0e29c0..b1e1e568d 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -39,10 +39,14 @@ class Contact < ActiveRecord::Base scope :current_registrars, ->(id) { where(registrar_id: id) } BIC = 'bic' + PRIV = 'priv' + BIRTHDAY = 'birthday' + PASSPORT = 'passport' + IDENT_TYPES = [ - BIC, # Company registry code (or similar) - 'priv', # National idendtification number - 'birthday' # Birthday date + BIC, # Company registry code (or similar) + PRIV, # National idendtification number + BIRTHDAY # Birthday date ] class << self From bd79a38230016dc46b30d8a5c8501c844b6a0d93 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 17:22:40 +0300 Subject: [PATCH 26/29] Added import readme --- lib/tasks/import.rake | 107 ++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 2b9e71cdf..08afc925d 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,4 +1,59 @@ namespace :import do + # README + # + # 1) ESTABLISH CONNECTION TO FRED DATABASE + # ---------------------------------------- + # + # Add 'fred' database connection settings to config/database.yml + # Example config: + # + # fred: + # host: localhost + # adapter: postgresql + # encoding: unicode + # pool: 5 + # username: fred + # password: fred + # + # Verify you have correctly connected to fred database: + # Open Rails console: + # + # cd your_registry_deploy_path/current/ + # RAILS_ENV=production bundle exec rails c + # in console: Legacy::Contact.last + # in console: exit + # + # In console you should get Last Legacy::Contact object. + # If you get any errors, scroll up and read first lines + # to figure out what went wrong to connect to fred database. + # + # + # 2) START IMPORT + # --------------- + # + # Import scrip does not write anything to fred database. + # Script is implemented this way, you can run it multiple times + # in case you need it. However already imported object are + # not reimported, thus if some object has been updated meanwhile + # in fred database, those updates will be missed and thous should + # be carried over manually. All new object in fred will be + # imported in multiple import script runs. + # + # Start all import: + # + # cd your_registry_deploy_path/current/ + # RAILS_ENV=production bundle exec rails import:all + # + # If you wish to import one by one, please follow individual import order + # from task 'Import all' tasks in this script. + + desc 'Import all' + task registrars: :environment do + Rake::Task['import:registrars'].invoke + Rake::Task['import:contacts'].invoke + Rake::Task['import:domains'].invoke + end + desc 'Import registrars' task registrars: :environment do start = Time.now.to_f @@ -53,10 +108,10 @@ namespace :import do # 6;"BIRTHDAY";"day of birth" ident_type_map = { - 2 => Contact::IDENT_PRIV, - 3 => Contact::IDENT_PASSPORT, - 4 => Contact::IDENT_TYPE_BIC, - 6 => Contact::IDENT_BIRTHDAY + 2 => Contact::PRIV, + 3 => Contact::PASSPORT, + 4 => Contact::BIC, + 6 => Contact::BIRTHDAY } contact_columns = %w( @@ -75,23 +130,14 @@ namespace :import do updator_str ident_country_code legacy_id - ) - - address_columns = %w( - city street + city zip - created_at - street2 - street3 - creator_str - updator_str - country_code state - legacy_contact_id + country_code ) - contacts, addresses = [], [] + contacts = [] existing_contact_ids = Contact.pluck(:legacy_id) user = "rake-#{`whoami`.strip} #{ARGV.join ' '}" count = 0 @@ -118,27 +164,17 @@ namespace :import do user, user, x.country.try(:strip), - x.id - ] - - addresses << [ + x.id, + [x.street1.try(:strip), x.street2.try(:strip), x.street3.try(:strip)].join('\n'), x.city.try(:strip), - x.street1.try(:strip), x.postalcode.try(:strip), - x.object_registry.try(:crdate), - x.street2.try(:strip), - x.street3.try(:strip), - user, - user, - x.country.try(:strip), x.stateorprovince.try(:strip), - x.id + x.country.try(:strip) ] if contacts.size % 10000 == 0 Contact.import contact_columns, contacts, validate: false - Address.import address_columns, addresses, validate: false - contacts, addresses = [], [] + contacts = [] end rescue => e puts "ERROR on index #{index}" @@ -147,17 +183,6 @@ namespace :import do end Contact.import contact_columns, contacts, validate: false - Address.import address_columns, addresses, validate: false - - puts '-----> Updating relations...' - ActiveRecord::Base.connection.execute( - "UPDATE addresses "\ - "SET contact_id = contacts.id "\ - "FROM contacts "\ - "WHERE contacts.legacy_id = legacy_contact_id "\ - "AND legacy_contact_id IS NOT NULL "\ - "AND contact_id IS NULL" - ) puts "-----> Imported #{count} new contacts in #{(Time.now.to_f - start).round(2)} seconds" end From 51714c6b329d1a106cfb28c4e7c631de31455456 Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 10:16:32 +0300 Subject: [PATCH 27/29] Handle domain whois body in sync --- app/models/domain.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/domain.rb b/app/models/domain.rb index 36092ed90..8222cea76 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -265,7 +265,7 @@ class Domain < ActiveRecord::Base update_column(whois_body: new_whois_body) update_whois_server end - handle_asynchronously :update_whois_body + # handle_asynchronously :update_whois_body # rubocop:enable Metrics/MethodLength def contacts_body From 9d91d5b2c0764f9cdd112e02b0498b8cec949efc Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 17:52:03 +0300 Subject: [PATCH 28/29] Rescue fred database config error --- app/models/legacy/db.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/legacy/db.rb b/app/models/legacy/db.rb index f960683d0..0c8e6624a 100644 --- a/app/models/legacy/db.rb +++ b/app/models/legacy/db.rb @@ -1,7 +1,11 @@ module Legacy class Db < ActiveRecord::Base self.abstract_class = true - establish_connection :fred + begin + establish_connection :fred + rescue ActiveRecord::AdapterNotSpecified => e + logger.info "'fred' database not configured, please update your database.yml file: #{e}" + end def readonly? true From e4a5d2a1af64dc96b19c080d2a11a2ac3c5f0fad Mon Sep 17 00:00:00 2001 From: Priit Tark Date: Thu, 2 Apr 2015 17:54:34 +0300 Subject: [PATCH 29/29] Typo fix --- lib/tasks/import.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 08afc925d..c53ec493c 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -48,7 +48,7 @@ namespace :import do # from task 'Import all' tasks in this script. desc 'Import all' - task registrars: :environment do + task all: :environment do Rake::Task['import:registrars'].invoke Rake::Task['import:contacts'].invoke Rake::Task['import:domains'].invoke