Merge pull request #2040 from internetee/1851-validate-estonian-company-data-from-business-registry

implement asynchronous rake task for collect contact data from csv

app/jobs/scan_csv_registry_businnes_contacts_job.rb

@@ -0,0 +1,21 @@
+class ScanCsvRegistryBusinnesContactsJob < ApplicationJob
+  def perform(filename)
+    BusinessRegistryContact.delete_all
+
+    return unless File.exist?(filename)
+
+    enumurate_csv_file(filename)
+  end
+
+  private
+
+  def enumurate_csv_file(filename)
+    CSV.foreach(filename, headers: true, col_sep: ';') do |row|
+      BusinessRegistryContact.create(
+        name: row[0],
+        registry_code: row[1],
+        status: row[5]
+      )
+    end
+  end
+end

app/models/business_registry_contact.rb

@@ -0,0 +1,2 @@
+class BusinessRegistryContact < ApplicationRecord
+end

db/migrate/20210616112332_create_business_registry_contacts.rb

@@ -0,0 +1,11 @@
+class CreateBusinessRegistryContacts < ActiveRecord::Migration[6.1]
+  def change
+    create_table :business_registry_contacts do |t|
+      t.string :name
+      t.string :registry_code
+      t.string :status
+
+      t.timestamps
+    end
+  end
+end

db/structure.sql

@@ -497,6 +497,39 @@ CREATE SEQUENCE public.bounced_mail_addresses_id_seq
 ALTER SEQUENCE public.bounced_mail_addresses_id_seq OWNED BY public.bounced_mail_addresses.id;
 
 
+--
+-- Name: business_registry_contacts; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.business_registry_contacts (
+    id bigint NOT NULL,
+    name character varying,
+    registry_code character varying,
+    status character varying,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
+--
+-- Name: business_registry_contacts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.business_registry_contacts_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: business_registry_contacts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.business_registry_contacts_id_seq OWNED BY public.business_registry_contacts.id;
+
+
 --
 -- Name: certificates; Type: TABLE; Schema: public; Owner: -
 --
@@ -541,7 +574,6 @@ ALTER SEQUENCE public.certificates_id_seq OWNED BY public.certificates.id;
 --
 
 CREATE TABLE public.contact_requests (
-
     id integer NOT NULL,
     whois_record_id integer NOT NULL,
     secret character varying NOT NULL,
@@ -553,7 +585,6 @@ CREATE TABLE public.contact_requests (
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
     message_id character varying
-
 );
 
 
@@ -575,6 +606,7 @@ CREATE SEQUENCE public.contact_requests_id_seq
 
 ALTER SEQUENCE public.contact_requests_id_seq OWNED BY public.contact_requests.id;
 
+
 --
 -- Name: contacts; Type: TABLE; Schema: public; Owner: -
 --
@@ -2766,6 +2798,13 @@ ALTER TABLE ONLY public.blocked_domains ALTER COLUMN id SET DEFAULT nextval('pub
 ALTER TABLE ONLY public.bounced_mail_addresses ALTER COLUMN id SET DEFAULT nextval('public.bounced_mail_addresses_id_seq'::regclass);
 
 
+--
+-- Name: business_registry_contacts id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.business_registry_contacts ALTER COLUMN id SET DEFAULT nextval('public.business_registry_contacts_id_seq'::regclass);
+
+
 --
 -- Name: certificates id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -3230,6 +3269,14 @@ ALTER TABLE ONLY public.bounced_mail_addresses
     ADD CONSTRAINT bounced_mail_addresses_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: business_registry_contacts business_registry_contacts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.business_registry_contacts
+    ADD CONSTRAINT business_registry_contacts_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: certificates certificates_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -5104,6 +5151,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20200916125326'),
 ('20200917104213'),
 ('20200921084356'),
-('20210215101019');
+('20210215101019'),
+('20210616112332');
 
 

lib/tasks/collect_invalid_validation_business_contacts.rake

@@ -0,0 +1,71 @@
+HEADERS = %w[domain id name code registrar].freeze
+
+namespace :contacts do
+  desc 'Starts collect invalid validation contacts'
+  task scan_org: :environment do
+    contacts = []
+
+    Contact.where(ident_type: 'org').each do |contact|
+      contacts << contact unless checking_contacts(contact)
+    end
+
+    magic_with_contacts(contacts)
+  end
+end
+
+def checking_contacts(contact)
+  c = BusinessRegistryContact.find_by(registry_code: contact.ident)
+  return false if c.nil? || c.status == 'N'
+
+  true
+end
+
+def magic_with_contacts(contacts)
+  CSV.open('invalid_business_contacts.csv', 'w') do |csv|
+    csv << HEADERS
+    contacts.each do |contact|
+      domains = domain_filter(contact)
+      domains.each do |domain|
+        registrar = Registrar.find_by(id: domain.registrar_id)
+        csv << [domain.name, contact.id, contact.name, contact.ident, registrar.name]
+      end
+    end
+  end
+end
+
+def domain_filter(contact)
+  domains = searching_domains(contact)
+  domains.reject! { |dom| dom.statuses.include? DomainStatus::FORCE_DELETE }
+  domains
+end
+
+def searching_domains(contact)
+  registrant_domains = Domain.where(registrant_id: contact.id)
+
+  tech_domains = collect_tech_domains(contact)
+  admin_domains = collect_admin_domains(contact)
+
+  tech_domains | admin_domains | registrant_domains
+end
+
+def collect_admin_domains(contact)
+  admin_domains = []
+
+  admin_contacts = AdminDomainContact.where(contact_id: contact.id)
+  admin_contacts.each do |c|
+    admin_domains << Domain.find(c.domain_id)
+  end
+
+  admin_domains
+end
+
+def collect_tech_domains(contact)
+  tech_domains = []
+
+  tech_contacts = TechDomainContact.where(contact_id: contact.id)
+  tech_contacts.each do |c|
+    tech_domains << Domain.find(c.domain_id)
+  end
+
+  tech_domains
+end

lib/tasks/collect_invalid_validation_contacts.rake

@@ -1,5 +1,5 @@
 namespace :collect_invalid_contacts do
-  desc 'Starts collect invalid validation contacts'
+  desc 'Starts collect invalid business contacts'
   task all_domains: :environment do
     prepare_csv_file
 

lib/tasks/scan_csv_registry_business_contacts.rake

@@ -0,0 +1,9 @@
+FILE_NAME = './ettevotja_rekvisiidid_init.csv'.freeze
+
+namespace :collect_csv_data do
+  desc 'Import from csv registry business contact into BusinessRegistryContact model'
+
+  task business_contacts: :environment do
+    ScanCsvRegistryBusinnesContactsJob.perform_later(FILE_NAME)
+  end
+end

test/fixtures/files/ette.csv

@@ -0,0 +1,10 @@
+nimi;ariregistri_kood;ettevotja_oiguslik_vorm;ettevotja_oigusliku_vormi_alaliik;kmkr_nr;ettevotja_staatus;ettevotja_staatus_tekstina;ettevotja_esmakande_kpv;ettevotja_aadress;asukoht_ettevotja_aadressis;asukoha_ehak_kood;asukoha_ehak_tekstina;indeks_ettevotja_aadressis;ads_adr_id;ads_ads_oid;ads_normaliseeritud_taisaadress;teabesysteemi_link
+001 group OÜ;12754230;Osaühing;;;R;Registrisse kantud;17.11.2014;;Õismäe tee 78-9;0176;Haabersti linnaosa, Tallinn, Harju maakond;13513;2182337;;Harju maakond, Tallinn, Haabersti linnaosa, Õismäe tee 78-9;https://ariregister.rik.ee/ettevotja.py?ark=12754230&ref=rekvisiidid
+001 Kinnisvara OÜ;12652512;Osaühing;;EE101721589;R;Registrisse kantud;25.04.2014;;Õismäe tee 78-9;0176;Haabersti linnaosa, Tallinn, Harju maakond;13513;2182337;;Harju maakond, Tallinn, Haabersti linnaosa, Õismäe tee 78-9;https://ariregister.rik.ee/ettevotja.py?ark=12652512&ref=rekvisiidid
+007 Autohaus osaühing;11694365;Osaühing;;EE101335276;R;Registrisse kantud;30.07.2009;;;;;;;;;https://ariregister.rik.ee/ettevotja.py?ark=11694365&ref=rekvisiidid
+013 investment OÜ;12937781;Osaühing;;;R;Registrisse kantud;29.10.2015;;Tartu mnt 18;0298;Kesklinna linnaosa, Tallinn, Harju maakond;10145;2311081;;Harju maakond, Tallinn, Kesklinna linnaosa, Tartu mnt 18;https://ariregister.rik.ee/ettevotja.py?ark=12937781&ref=rekvisiidid
+01Arvutiabi OÜ;14112620;Osaühing;;;R;Registrisse kantud;13.09.2016;;Aru tn 19-5;0614;Põhja-Tallinna linnaosa, Tallinn, Harju maakond;10318;2283769;;Harju maakond, Tallinn, Põhja-Tallinna linnaosa, Aru tn 19-5;https://ariregister.rik.ee/ettevotja.py?ark=14112620&ref=rekvisiidid
+01 Creations Osaühing;10818150;Osaühing;;;R;Registrisse kantud;21.11.2001;;Ahtri tn 12;0298;Kesklinna linnaosa, Tallinn, Harju maakond;10151;2113048;;Harju maakond, Tallinn, Kesklinna linnaosa, Ahtri tn 12;https://ariregister.rik.ee/ettevotja.py?ark=10818150&ref=rekvisiidid
+020 EHITUS OÜ;16035029;Osaühing;;EE102307287;R;Registrisse kantud;24.08.2020;;Harju maakond, Tallinn, Kristiine linnaosa, A. H. Tammsaare tee 47;0339;Kristiine linnaosa, Tallinn, Harju maakond;11316;2216417;;Harju maakond, Tallinn, Kristiine linnaosa, A. H. Tammsaare tee 47;https://ariregister.rik.ee/ettevotja.py?ark=16035029&ref=rekvisiidid
+02JDM OÜ;16240491;Osaühing;;;R;Registrisse kantud;28.05.2021;;Mustamäe tee 106-20;0482;Mustamäe linnaosa, Tallinn, Harju maakond;12917;2311996;;Harju maakond, Tallinn, Mustamäe linnaosa, Mustamäe tee 106-20;https://ariregister.rik.ee/ettevotja.py?ark=16240491&ref=rekvisiidid
+0dav riie OÜ;14288602;Osaühing;;;R;Registrisse kantud;04.07.2017;;Vanaaseme;2087;Ingliste küla, Kehtna vald, Rapla maakond;79004;3201213;;Rapla maakond, Kehtna vald, Ingliste küla, Vanaaseme;https://ariregister.rik.ee/ettevotja.py?ark=14288602&ref=rekvisiidid

test/jobs/scan_csv_registry_businnes_contacts_job_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class ScanCsvRegistryBusinnesContactsJobTest < ActiveJob::TestCase
+  def test_delivers_email
+    filename = 'test/fixtures/files/ette.csv'
+
+    assert_performed_jobs 1 do
+      ScanCsvRegistryBusinnesContactsJob.perform_later(filename)
+    end
+  end
+end