Find in batches for contact import

This commit is contained in:
Martin Lensment 2015-03-02 11:40:40 +02:00 committed by Priit Tark
parent 9cd4bad3a9
commit 20f8192f78
2 changed files with 16 additions and 4 deletions

View file

@ -3,5 +3,6 @@ module Legacy
self.table_name = :contact self.table_name = :contact
belongs_to :object_registry, foreign_key: :id belongs_to :object_registry, foreign_key: :id
belongs_to :object, foreign_key: :id belongs_to :object, foreign_key: :id
belongs_to :registrar, foreign_key: :crid
end end
end end

View file

@ -40,6 +40,7 @@ namespace :import do
desc 'Import contacts' desc 'Import contacts'
task contacts: :environment do task contacts: :environment do
start = Time.now.to_i
puts '-----> Importing contacts...' puts '-----> Importing contacts...'
contacts = [] contacts = []
@ -59,10 +60,12 @@ namespace :import do
6 => Contact::IDENT_BIRTHDAY 6 => Contact::IDENT_BIRTHDAY
} }
Legacy::Contact.all.each do |x| contacts = []
Legacy::Contact.includes(:object_registry, :object, :registrar).find_each(batch_size: 10000) do |x|
next if existing_ids.include?(x.id) next if existing_ids.include?(x.id)
begin begin
registrar = Registrar.find_by(legacy_id: x.object_registry.crid) # registrar = Registrar.find_by(legacy_id: x.object_registry.crid)
contacts << Contact.new({ contacts << Contact.new({
code: x.object_registry.name, code: x.object_registry.name,
@ -78,7 +81,7 @@ namespace :import do
auth_info: x.object.authinfopw.try(:strip), auth_info: x.object.authinfopw.try(:strip),
name: x.name.try(:strip), name: x.name.try(:strip),
org_name: x.organization.try(:strip), org_name: x.organization.try(:strip),
registrar_id: registrar.try(:id), registrar_id: x.registrar.try(:id),
creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", creator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}",
updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}", updator_str: "rake-#{`whoami`.strip} #{ARGV.join ' '}",
ident_country_code: x.country.try(:strip), ident_country_code: x.country.try(:strip),
@ -96,8 +99,16 @@ namespace :import do
state: x.stateorprovince.try(:strip) state: x.stateorprovince.try(:strip)
}) })
}) })
if contacts.size % 10000 == 0
puts "commtting #{contacts.size}"
puts Time.now.to_i - start
# Contact.import contacts, validate: false
contacts = []
end
rescue => e rescue => e
binding.pry puts e
puts x.inspect
end end
end end