From 225437cdb44ff7cb300b5b004fa0a95a1d4cceef Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:52:35 -0600 Subject: [PATCH 01/88] Add fields --- .gitignore | 3 ++ ...ransitiondomain_federal_agency_and_more.py | 34 +++++++++++++++++++ src/registrar/models/transition_domain.py | 18 +++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py diff --git a/.gitignore b/.gitignore index a4df194fa..449723b77 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,6 @@ node_modules # Compliance/trestle related docs/compliance/.trestle/cache +src/migrationdata/20231009.domaintypes.adhoc.dotgov.txt +src/migrationdata/20231009.domainadditionaldatalink.adhoc.dotgov.txt +src/migrationdata/20231009.agency.adhoc.dotgov.txt diff --git a/src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py b/src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py new file mode 100644 index 000000000..361b91789 --- /dev/null +++ b/src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2.1 on 2023-10-24 20:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0042_create_groups_v03"), + ] + + operations = [ + migrations.AddField( + model_name="transitiondomain", + name="federal_agency", + field=models.TextField(blank=True, help_text="Federal agency", null=True), + ), + migrations.AddField( + model_name="transitiondomain", + name="federal_type", + field=models.TextField( + blank=True, + help_text="Federal government branch", + max_length=50, + null=True, + ), + ), + migrations.AddField( + model_name="transitiondomain", + name="organization_type", + field=models.TextField( + blank=True, help_text="Type of organization", max_length=255, null=True + ), + ), + ] diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 232fd9033..d95d8e441 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -1,5 +1,4 @@ from django.db import models - from .utility.time_stamped_model import TimeStampedModel @@ -43,6 +42,23 @@ class TransitionDomain(TimeStampedModel): verbose_name="email sent", help_text="indicates whether email was sent", ) + organization_type = models.TextField( + max_length=255, + null=True, + blank=True, + help_text="Type of organization", + ) + federal_type = models.TextField( + max_length=50, + null=True, + blank=True, + help_text="Federal government branch", + ) + federal_agency = models.TextField( + null=True, + blank=True, + help_text="Federal agency", + ) def __str__(self): return ( From afcb0ec15abd8575fb5e87175897788060fc060d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 26 Oct 2023 12:03:35 -0600 Subject: [PATCH 02/88] Changes --- .../commands/load_extra_transition_domain.py | 78 ++++++++++++ .../commands/load_transition_domain.py | 7 ++ .../commands/utility/epp_data_containers.py | 41 +++++++ .../utility/extra_transition_domain.py | 116 ++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 src/registrar/management/commands/load_extra_transition_domain.py create mode 100644 src/registrar/management/commands/utility/epp_data_containers.py create mode 100644 src/registrar/management/commands/utility/extra_transition_domain.py diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py new file mode 100644 index 000000000..77ca55291 --- /dev/null +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -0,0 +1,78 @@ +"""""" +import csv +import glob +import re +import logging + +import os +from typing import List +from enum import Enum +from django.core.management import BaseCommand +from .utility.extra_transition_domain import ExtraTransitionDomain + + +logger = logging.getLogger(__name__) + +class EnumFilenames(Enum): + AGENCY_ADHOC = "agency.adhoc.dotgov.txt" + DOMAIN_ADDITIONAL = "domainadditionaldatalink.adhoc.dotgov.txt" + DOMAIN_ADHOC = "domaintypes.adhoc.dotgov.txt" + ORGANIZATION_ADHOC = "organization.adhoc.dotgov.txt" + +class Command(BaseCommand): + help = "" + + filenames = EnumFilenames + + strip_date_regex = re.compile(r'\d+\.(.+)') + # While the prefix of these files typically includes the date, + # the rest of them following a predefined pattern. Define this here, + # and search for that to infer what is wanted. + filename_pattern_mapping = { + # filename - regex to use when encountered + filenames.AGENCY_ADHOC: strip_date_regex, + filenames.DOMAIN_ADDITIONAL: strip_date_regex, + filenames.DOMAIN_ADHOC: strip_date_regex, + filenames.ORGANIZATION_ADHOC: strip_date_regex + } + + def add_arguments(self, parser): + """Add filename arguments.""" + parser.add_argument( + "--directory", + default="migrationdata", + help="Desired directory" + ) + parser.add_argument( + "--agency_adhoc_filename", + default=self.filenames.AGENCY_ADHOC, + help="Defines the filename for agency adhocs", + ) + parser.add_argument( + "--domain_additional_filename", + default=self.filenames.DOMAIN_ADDITIONAL, + help="Defines the filename for additional domain data", + ) + parser.add_argument( + "--domain_adhoc_filename", + default=self.filenames.DOMAIN_ADHOC, + help="Defines the filename for domain type adhocs", + ) + parser.add_argument( + "--organization_adhoc_filename", + default=self.filenames.ORGANIZATION_ADHOC, + help="Defines the filename for domain type adhocs", + ) + parser.add_argument("--sep", default="|", help="Delimiter character") + + def handle(self, *args, **options): + self.data = ExtraTransitionDomain( + agency_adhoc_filename=options['agency_adhoc_filename'], + domain_additional_filename=options['domain_additional_filename'], + domain_adhoc_filename=options['domain_adhoc_filename'], + organization_adhoc_filename=options['organization_adhoc_filename'], + directory=options['directory'], + seperator=options['sep'] + ) + + diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 206589c33..624418fe9 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -88,6 +88,9 @@ class Command(BaseCommand): parser.add_argument( "domain_statuses_filename", help="Data file with domain status information" ) + parser.add_argument( + "--loadExtraData", default=True, help="Determines if additional metadata should be applied" + ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -306,6 +309,7 @@ class Command(BaseCommand): ): """Parse the data files and create TransitionDomains.""" sep = options.get("sep") + load_extra_data = options.get("loadExtraData") # If --resetTable was used, prompt user to confirm # deletion of table data @@ -322,6 +326,9 @@ class Command(BaseCommand): # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + + if load_extra_data: + # STEP 1: # Create mapping of domain name -> status diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py new file mode 100644 index 000000000..8bc7a9b4d --- /dev/null +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from typing import Optional + +@dataclass +class AgencyAdhoc(): + """Defines the structure given in the given AGENCY_ADHOC file""" + agencyid: Optional[int] = None + agencyname: Optional[str] = None + active: Optional[bool] = None + isfederal: Optional[bool] = None + + +@dataclass +class DomainAdditionalData(): + """Defines the structure given in the given DOMAIN_ADDITIONAL file""" + domainname: Optional[str] = None + domaintypeid: Optional[int] = None + authorityid: Optional[int] = None + orgid: Optional[int] = None + securitycontact_email: Optional[str] = None + dnsseckeymonitor: Optional[str] = None + domainpurpose: Optional[str] = None + +@dataclass +class DomainTypeAdhoc(): + """Defines the structure given in the given DOMAIN_ADHOC file""" + domaintypeid: Optional[int] = None + domaintype: Optional[str] = None + code: Optional[str] = None + active: Optional[bool] = None + +@dataclass +class OrganizationAdhoc(): + """Defines the structure given in the given ORGANIZATION_ADHOC file""" + orgid: Optional[int] = None + orgname: Optional[str] = None + orgstreet: Optional[str] = None + orgcity: Optional[str] = None + orgstate: Optional[str] = None + orgzip: Optional[str] = None + orgcountrycode: Optional[str] = None \ No newline at end of file diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py new file mode 100644 index 000000000..9b9c27ac6 --- /dev/null +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -0,0 +1,116 @@ +"""""" +import csv +from dataclasses import dataclass +import glob +import re +import logging + +import os +from typing import List +from enum import Enum +from epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc + +logger = logging.getLogger(__name__) + +class EnumFilenames(Enum): + AGENCY_ADHOC = "agency.adhoc.dotgov.txt" + DOMAIN_ADDITIONAL = "domainadditionaldatalink.adhoc.dotgov.txt" + DOMAIN_ADHOC = "domaintypes.adhoc.dotgov.txt" + ORGANIZATION_ADHOC = "organization.adhoc.dotgov.txt" + +@dataclass +class PatternMap(): + def __init__(self, filename, regex, datatype): + self.filename = filename + self.regex = regex + self.datatype = datatype + + +class ExtraTransitionDomain(): + filenames = EnumFilenames + strip_date_regex = re.compile(r'\d+\.(.+)') + filename_pattern_mapping = { + # filename - regex to use when encountered + filenames.AGENCY_ADHOC: strip_date_regex, + filenames.DOMAIN_ADDITIONAL: strip_date_regex, + filenames.DOMAIN_ADHOC: strip_date_regex, + filenames.ORGANIZATION_ADHOC: strip_date_regex + } + + def __init__(self, + agency_adhoc_filename=filenames.AGENCY_ADHOC, + domain_additional_filename=filenames.DOMAIN_ADDITIONAL, + domain_adhoc_filename=filenames.DOMAIN_ADHOC, + organization_adhoc_filename=filenames.ORGANIZATION_ADHOC, + directory="migrationdata", + seperator="|" + ): + self.directory = directory + self.seperator = seperator + self.all_files = glob.glob(f"{directory}/*") + self.filename_dicts = [] + + self.agency_adhoc: List[AgencyAdhoc] = [] + self.domain_additional: List[DomainAdditionalData] = [] + self.domain_adhoc: List[DomainTypeAdhoc] = [] + self.organization_adhoc: List[OrganizationAdhoc] = [] + + # Generate filename dictionaries + for filename, enum_pair in [ + (agency_adhoc_filename, self.filenames.AGENCY_ADHOC), + (domain_additional_filename, self.filenames.DOMAIN_ADDITIONAL), + (domain_adhoc_filename, self.filenames.DOMAIN_ADHOC), + (organization_adhoc_filename, self.filenames.ORGANIZATION_ADHOC) + ]: + # Generates a dictionary that associates the enum type to + # the requested filename, and checks if its the default type. + self.filename_dicts.append(self._create_filename_dict(filename, enum_pair)) + + def parse_all_files(self, seperator): + for file in self.all_files: + filename = os.path.basename(file) + for item in self.filename_dicts: + if filename == item.get("filename"): + match item.get("default_filename"): + case self.filenames.AGENCY_ADHOC: + self.agency_adhoc = self._read_csv_file(filename, seperator, AgencyAdhoc) + case self.filenames.DOMAIN_ADDITIONAL: + self.domain_additional = self._read_csv_file(filename, seperator, DomainAdditionalData) + case self.filenames.DOMAIN_ADHOC: + self.domain_adhoc = self._read_csv_file(filename, seperator, DomainTypeAdhoc) + case self.filenames.ORGANIZATION_ADHOC: + self.organization_adhoc = self._read_csv_file(filename, seperator, OrganizationAdhoc) + case _: + logger.warning("Could not find default mapping") + break + + def _read_csv_file(self, file, seperator, dataclass_type): + with open(file, "r", encoding="utf-8") as requested_file: + reader = csv.DictReader(requested_file, delimiter=seperator) + return [dataclass_type(**row) for row in reader] + + + def _create_filename_dict(self, filename, default_filename): + regex = self.filename_pattern_mapping.get(filename) + + # returns (filename, inferred_successfully) + infer = self._infer_filename(regex, filename) + filename_dict = { + "filename": infer[0], + "default_filename": default_filename, + "is_default": filename == default_filename, + "could_infer": infer[1] + } + return filename_dict + + def _infer_filename(self, regex, current_file_name): + if regex is None: + return (current_file_name, False) + + match = regex.match(current_file_name) + + if match is None: + return (None, False) + + filename_without_date = match.group(1) + return (match, filename_without_date == current_file_name) From 59c77de91e8fc40d02c3e3a2ffbe2f299c1faba0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 26 Oct 2023 12:03:55 -0600 Subject: [PATCH 03/88] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 449723b77..7c8e4e884 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,4 @@ docs/compliance/.trestle/cache src/migrationdata/20231009.domaintypes.adhoc.dotgov.txt src/migrationdata/20231009.domainadditionaldatalink.adhoc.dotgov.txt src/migrationdata/20231009.agency.adhoc.dotgov.txt +src/migrationdata/20231009.organization.adhoc.dotgov.txt From 00f44f2f8488dc712cd5a6cc5fa12bcb56bca17c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:43:39 -0600 Subject: [PATCH 04/88] Changes --- .../commands/load_extra_transition_domain.py | 25 +-- .../utility/extra_transition_domain.py | 146 +++++++++--------- 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index 77ca55291..307a3b3c7 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -65,14 +65,19 @@ class Command(BaseCommand): ) parser.add_argument("--sep", default="|", help="Delimiter character") - def handle(self, *args, **options): - self.data = ExtraTransitionDomain( - agency_adhoc_filename=options['agency_adhoc_filename'], - domain_additional_filename=options['domain_additional_filename'], - domain_adhoc_filename=options['domain_adhoc_filename'], - organization_adhoc_filename=options['organization_adhoc_filename'], - directory=options['directory'], - seperator=options['sep'] - ) - + def handle(self, **options): + try: + self.domain_object = ExtraTransitionDomain( + agency_adhoc_filename=options['agency_adhoc_filename'], + domain_additional_filename=options['domain_additional_filename'], + domain_adhoc_filename=options['domain_adhoc_filename'], + organization_adhoc_filename=options['organization_adhoc_filename'], + directory=options['directory'], + seperator=options['sep'] + ) + self.domain_object.parse_all_files() + except Exception as err: + logger.error(f"Could not load additional data. Error: {err}") + else: + diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index 9b9c27ac6..39dde48b2 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -13,104 +13,98 @@ from epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdh logger = logging.getLogger(__name__) class EnumFilenames(Enum): - AGENCY_ADHOC = "agency.adhoc.dotgov.txt" - DOMAIN_ADDITIONAL = "domainadditionaldatalink.adhoc.dotgov.txt" - DOMAIN_ADHOC = "domaintypes.adhoc.dotgov.txt" - ORGANIZATION_ADHOC = "organization.adhoc.dotgov.txt" + """Returns a tuple mapping for (filetype, default_file_name). + + For instance, AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") + """ + AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") + DOMAIN_ADDITIONAL = ("domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt") + DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") + ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") @dataclass class PatternMap(): - def __init__(self, filename, regex, datatype): - self.filename = filename - self.regex = regex - self.datatype = datatype + def __init__(self, filename: str, regex, data_type, data=[]): + self.regex = regex + self.data_type = data_type + self.data = data + + # returns (filename, inferred_successfully) + _infer = self._infer_filename(self.regex, filename) + self.filename = _infer[0] + self.could_infer = _infer[1] + + + def _infer_filename(self, regex, default_file_name): + if not isinstance(regex, re.Pattern): + return (self.filename, False) + + match = regex.match(self.filename) + if not match: + return (self.filename, False) + + date = match.group(1) + filename_without_date = match.group(2) + + can_infer = filename_without_date == default_file_name + if not can_infer: + return (self.filename, False) + + full_filename = date + filename_without_date + return (full_filename, can_infer) class ExtraTransitionDomain(): filenames = EnumFilenames strip_date_regex = re.compile(r'\d+\.(.+)') - filename_pattern_mapping = { - # filename - regex to use when encountered - filenames.AGENCY_ADHOC: strip_date_regex, - filenames.DOMAIN_ADDITIONAL: strip_date_regex, - filenames.DOMAIN_ADHOC: strip_date_regex, - filenames.ORGANIZATION_ADHOC: strip_date_regex - } def __init__(self, - agency_adhoc_filename=filenames.AGENCY_ADHOC, - domain_additional_filename=filenames.DOMAIN_ADDITIONAL, - domain_adhoc_filename=filenames.DOMAIN_ADHOC, - organization_adhoc_filename=filenames.ORGANIZATION_ADHOC, + agency_adhoc_filename=filenames.AGENCY_ADHOC[1], + domain_additional_filename=filenames.DOMAIN_ADDITIONAL[1], + domain_adhoc_filename=filenames.DOMAIN_ADHOC[1], + organization_adhoc_filename=filenames.ORGANIZATION_ADHOC[1], directory="migrationdata", seperator="|" ): self.directory = directory self.seperator = seperator self.all_files = glob.glob(f"{directory}/*") - self.filename_dicts = [] + # Create a set with filenames as keys for quick lookup + self.all_files_set = {os.path.basename(file) for file in self.all_files} - self.agency_adhoc: List[AgencyAdhoc] = [] - self.domain_additional: List[DomainAdditionalData] = [] - self.domain_adhoc: List[DomainTypeAdhoc] = [] - self.organization_adhoc: List[OrganizationAdhoc] = [] + self.csv_data = { + self.filenames.AGENCY_ADHOC: PatternMap(agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc), + self.filenames.DOMAIN_ADDITIONAL: PatternMap(domain_additional_filename, self.strip_date_regex, DomainAdditionalData), + self.filenames.DOMAIN_ADHOC: PatternMap(domain_adhoc_filename, self.strip_date_regex, DomainTypeAdhoc), + self.filenames.ORGANIZATION_ADHOC: PatternMap(organization_adhoc_filename, self.strip_date_regex, OrganizationAdhoc) + } - # Generate filename dictionaries - for filename, enum_pair in [ - (agency_adhoc_filename, self.filenames.AGENCY_ADHOC), - (domain_additional_filename, self.filenames.DOMAIN_ADDITIONAL), - (domain_adhoc_filename, self.filenames.DOMAIN_ADHOC), - (organization_adhoc_filename, self.filenames.ORGANIZATION_ADHOC) - ]: - # Generates a dictionary that associates the enum type to - # the requested filename, and checks if its the default type. - self.filename_dicts.append(self._create_filename_dict(filename, enum_pair)) - def parse_all_files(self, seperator): - for file in self.all_files: - filename = os.path.basename(file) - for item in self.filename_dicts: - if filename == item.get("filename"): - match item.get("default_filename"): - case self.filenames.AGENCY_ADHOC: - self.agency_adhoc = self._read_csv_file(filename, seperator, AgencyAdhoc) - case self.filenames.DOMAIN_ADDITIONAL: - self.domain_additional = self._read_csv_file(filename, seperator, DomainAdditionalData) - case self.filenames.DOMAIN_ADHOC: - self.domain_adhoc = self._read_csv_file(filename, seperator, DomainTypeAdhoc) - case self.filenames.ORGANIZATION_ADHOC: - self.organization_adhoc = self._read_csv_file(filename, seperator, OrganizationAdhoc) - case _: - logger.warning("Could not find default mapping") - break + def parse_all_files(self): + """Clears all preexisting data then parses each related CSV file""" + self.clear_csv_data() + for item in self.csv_data: + file_type: PatternMap = item.value + filename = file_type.filename + + if filename in self.all_files_set: + file_type.data = self._read_csv_file( + self.all_files_set[filename], + self.seperator, + file_type.data_type + ) + else: + # Log if we can't find the desired file + logger.warning(f"Could not find file: {filename}") + + + def clear_csv_data(self): + for item in self.csv_data: + file_type: PatternMap = item.value + file_type.data = [] def _read_csv_file(self, file, seperator, dataclass_type): with open(file, "r", encoding="utf-8") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) return [dataclass_type(**row) for row in reader] - - def _create_filename_dict(self, filename, default_filename): - regex = self.filename_pattern_mapping.get(filename) - - # returns (filename, inferred_successfully) - infer = self._infer_filename(regex, filename) - filename_dict = { - "filename": infer[0], - "default_filename": default_filename, - "is_default": filename == default_filename, - "could_infer": infer[1] - } - return filename_dict - - def _infer_filename(self, regex, current_file_name): - if regex is None: - return (current_file_name, False) - - match = regex.match(current_file_name) - - if match is None: - return (None, False) - - filename_without_date = match.group(1) - return (match, filename_without_date == current_file_name) From 519595ba9b0fdb217fc6af5344a493f932031ee1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:29:50 -0600 Subject: [PATCH 05/88] Migration change, script work --- .gitignore | 1 + .../commands/load_extra_transition_domain.py | 57 +++++++++-------- .../commands/utility/epp_data_containers.py | 13 +++- .../utility/extra_transition_domain.py | 61 +++++++++++-------- ...0044_transitiondomain_organization_name.py | 19 ++++++ src/registrar/models/transition_domain.py | 5 ++ 6 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 src/registrar/migrations/0044_transitiondomain_organization_name.py diff --git a/.gitignore b/.gitignore index 7c8e4e884..48b6af579 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,4 @@ src/migrationdata/20231009.domaintypes.adhoc.dotgov.txt src/migrationdata/20231009.domainadditionaldatalink.adhoc.dotgov.txt src/migrationdata/20231009.agency.adhoc.dotgov.txt src/migrationdata/20231009.organization.adhoc.dotgov.txt +src/migrationdata/20231009.organization.adhoc.dotgov.txt diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index 307a3b3c7..866508fb1 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -8,34 +8,18 @@ import os from typing import List from enum import Enum from django.core.management import BaseCommand -from .utility.extra_transition_domain import ExtraTransitionDomain +from registrar.models.transition_domain import TransitionDomain +from .utility.extra_transition_domain import ExtraTransitionDomain +from .utility.epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, EnumFilenames logger = logging.getLogger(__name__) -class EnumFilenames(Enum): - AGENCY_ADHOC = "agency.adhoc.dotgov.txt" - DOMAIN_ADDITIONAL = "domainadditionaldatalink.adhoc.dotgov.txt" - DOMAIN_ADHOC = "domaintypes.adhoc.dotgov.txt" - ORGANIZATION_ADHOC = "organization.adhoc.dotgov.txt" class Command(BaseCommand): help = "" - filenames = EnumFilenames - strip_date_regex = re.compile(r'\d+\.(.+)') - # While the prefix of these files typically includes the date, - # the rest of them following a predefined pattern. Define this here, - # and search for that to infer what is wanted. - filename_pattern_mapping = { - # filename - regex to use when encountered - filenames.AGENCY_ADHOC: strip_date_regex, - filenames.DOMAIN_ADDITIONAL: strip_date_regex, - filenames.DOMAIN_ADHOC: strip_date_regex, - filenames.ORGANIZATION_ADHOC: strip_date_regex - } - def add_arguments(self, parser): """Add filename arguments.""" parser.add_argument( @@ -45,22 +29,22 @@ class Command(BaseCommand): ) parser.add_argument( "--agency_adhoc_filename", - default=self.filenames.AGENCY_ADHOC, + default=self.filenames.AGENCY_ADHOC[1], help="Defines the filename for agency adhocs", ) parser.add_argument( "--domain_additional_filename", - default=self.filenames.DOMAIN_ADDITIONAL, + default=self.filenames.DOMAIN_ADDITIONAL[1], help="Defines the filename for additional domain data", ) parser.add_argument( "--domain_adhoc_filename", - default=self.filenames.DOMAIN_ADHOC, + default=self.filenames.DOMAIN_ADHOC[1], help="Defines the filename for domain type adhocs", ) parser.add_argument( "--organization_adhoc_filename", - default=self.filenames.ORGANIZATION_ADHOC, + default=self.filenames.ORGANIZATION_ADHOC[1], help="Defines the filename for domain type adhocs", ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -79,5 +63,30 @@ class Command(BaseCommand): except Exception as err: logger.error(f"Could not load additional data. Error: {err}") else: - + for transition_domain in TransitionDomain.objects.all(): + transition_domain.organization_type + def get_organization_adhoc(self, desired_id): + """Grabs adhoc information for organizations. Returns an organization + dictionary + + returns: + { + " + } + """ + return self.get_object_by_id(self.filenames.ORGANIZATION_ADHOC, desired_id) + + def get_domain_adhoc(self, desired_id): + """""" + return self.get_object_by_id(self.filenames.DOMAIN_ADHOC, desired_id) + + def get_agency_adhoc(self, desired_id): + """""" + return self.get_object_by_id(self.filenames.AGENCY_ADHOC, desired_id) + + def get_object_by_id(self, file_type: EnumFilenames, desired_id): + """""" + desired_type = self.domain_object.csv_data.get(file_type) + obj = desired_type.get(desired_id) + return obj diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 8bc7a9b4d..2d006c42b 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from enum import Enum from typing import Optional @dataclass @@ -38,4 +39,14 @@ class OrganizationAdhoc(): orgcity: Optional[str] = None orgstate: Optional[str] = None orgzip: Optional[str] = None - orgcountrycode: Optional[str] = None \ No newline at end of file + orgcountrycode: Optional[str] = None + +class EnumFilenames(Enum): + """Returns a tuple mapping for (filetype, default_file_name). + + For instance, AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") + """ + AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") + DOMAIN_ADDITIONAL = ("domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt") + DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") + ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") \ No newline at end of file diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index 39dde48b2..a54540960 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -7,27 +7,34 @@ import logging import os from typing import List -from enum import Enum -from epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc +from epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, EnumFilenames logger = logging.getLogger(__name__) -class EnumFilenames(Enum): - """Returns a tuple mapping for (filetype, default_file_name). - - For instance, AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") - """ - AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") - DOMAIN_ADDITIONAL = ("domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt") - DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") - ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") @dataclass class PatternMap(): + """Helper class that holds data and metadata about a requested file. + + filename: str -> The desired filename to target. If no filename is given, + it is assumed that you are passing in a filename pattern and it will look + for a filename that matches the given postfix you pass in. - def __init__(self, filename: str, regex, data_type, data=[]): + regex: re.Pattern -> Defines what regex you want to use when inferring + filenames. If none, no matching occurs. + + data_type: type -> Metadata about the desired type for data. + + id_field: str -> Defines which field should act as the id in data. + + data: dict -> The returned data. Intended to be used with data_type + to cross-reference. + + """ + def __init__(self, filename: str, regex: re.Pattern, data_type: type, id_field: str, data: dict = {}): self.regex = regex self.data_type = data_type + self.id_field = id_field self.data = data # returns (filename, inferred_successfully) @@ -36,7 +43,7 @@ class PatternMap(): self.could_infer = _infer[1] - def _infer_filename(self, regex, default_file_name): + def _infer_filename(self, regex: re.Pattern, default_file_name): if not isinstance(regex, re.Pattern): return (self.filename, False) @@ -73,15 +80,20 @@ class ExtraTransitionDomain(): self.all_files_set = {os.path.basename(file) for file in self.all_files} self.csv_data = { - self.filenames.AGENCY_ADHOC: PatternMap(agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc), - self.filenames.DOMAIN_ADDITIONAL: PatternMap(domain_additional_filename, self.strip_date_regex, DomainAdditionalData), - self.filenames.DOMAIN_ADHOC: PatternMap(domain_adhoc_filename, self.strip_date_regex, DomainTypeAdhoc), - self.filenames.ORGANIZATION_ADHOC: PatternMap(organization_adhoc_filename, self.strip_date_regex, OrganizationAdhoc) + # (filename, default_url): metadata about the desired file + self.filenames.AGENCY_ADHOC: PatternMap(agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc, "agencyid"), + self.filenames.DOMAIN_ADDITIONAL: PatternMap(domain_additional_filename, self.strip_date_regex, DomainAdditionalData, "domainname"), + self.filenames.DOMAIN_ADHOC: PatternMap(domain_adhoc_filename, self.strip_date_regex, DomainTypeAdhoc, "domaintypeid"), + self.filenames.ORGANIZATION_ADHOC: PatternMap(organization_adhoc_filename, self.strip_date_regex, OrganizationAdhoc, "orgid") } - def parse_all_files(self): - """Clears all preexisting data then parses each related CSV file""" + def parse_all_files(self, overwrite_existing_data = True): + """Clears all preexisting data then parses each related CSV file. + + overwrite_existing_data: bool -> Determines if we should clear + csv_data.data if it already exists + """ self.clear_csv_data() for item in self.csv_data: file_type: PatternMap = item.value @@ -91,20 +103,21 @@ class ExtraTransitionDomain(): file_type.data = self._read_csv_file( self.all_files_set[filename], self.seperator, - file_type.data_type + file_type.data_type, + file_type.id_field ) else: # Log if we can't find the desired file - logger.warning(f"Could not find file: {filename}") + logger.error(f"Could not find file: {filename}") def clear_csv_data(self): for item in self.csv_data: file_type: PatternMap = item.value - file_type.data = [] + file_type.data = {} - def _read_csv_file(self, file, seperator, dataclass_type): + def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) - return [dataclass_type(**row) for row in reader] + return {row[id_field]: dataclass_type(**row) for row in reader} diff --git a/src/registrar/migrations/0044_transitiondomain_organization_name.py b/src/registrar/migrations/0044_transitiondomain_organization_name.py new file mode 100644 index 000000000..07c6ece82 --- /dev/null +++ b/src/registrar/migrations/0044_transitiondomain_organization_name.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.6 on 2023-10-27 14:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0043_transitiondomain_federal_agency_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="transitiondomain", + name="organization_name", + field=models.TextField( + blank=True, help_text="Organization name", null=True + ), + ), + ] diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index d95d8e441..6670b7bbd 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -48,6 +48,11 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Type of organization", ) + organization_name = models.TextField( + null=True, + blank=True, + help_text="Organization name", + ) federal_type = models.TextField( max_length=50, null=True, From fd4809ec66a560cd7184dcf182198210243872cb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:40:17 -0600 Subject: [PATCH 06/88] Update .gitignore --- .gitignore | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 48b6af579..ddd75475d 100644 --- a/.gitignore +++ b/.gitignore @@ -171,8 +171,6 @@ node_modules # Compliance/trestle related docs/compliance/.trestle/cache -src/migrationdata/20231009.domaintypes.adhoc.dotgov.txt -src/migrationdata/20231009.domainadditionaldatalink.adhoc.dotgov.txt -src/migrationdata/20231009.agency.adhoc.dotgov.txt -src/migrationdata/20231009.organization.adhoc.dotgov.txt -src/migrationdata/20231009.organization.adhoc.dotgov.txt + +src/migrationdata/* + From 24cb865ee3dfcf285b31c898c500bfbff5ebd434 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:30:44 -0600 Subject: [PATCH 07/88] Parsing logic --- .../commands/cat_files_into_getgov.py | 28 +- .../commands/load_extra_transition_domain.py | 334 ++++++++++++++++-- .../commands/load_transition_domain.py | 6 - .../commands/test_domain_migration.py | 195 ++++++---- .../commands/utility/epp_data_containers.py | 42 ++- .../utility/extra_transition_domain.py | 83 +++-- .../commands/utility/terminal_helper.py | 5 +- ...0044_transitiondomain_organization_name.py | 19 - src/registrar/models/transition_domain.py | 5 - 9 files changed, 526 insertions(+), 191 deletions(-) delete mode 100644 src/registrar/migrations/0044_transitiondomain_organization_name.py diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 17964f236..cb00e9f31 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -21,17 +21,19 @@ class Command(BaseCommand): default="txt", help="What file extensions to look for, like txt or gz", ) - parser.add_argument("--directory", default="migrationdata", help="Desired directory") + parser.add_argument( + "--directory", default="migrationdata", help="Desired directory" + ) def handle(self, **options): - file_extension: str = options.get("file_extension").lstrip('.') + file_extension: str = options.get("file_extension").lstrip(".") directory = options.get("directory") # file_extension is always coerced as str, Truthy is OK to use here. if not file_extension or not isinstance(file_extension, str): raise ValueError(f"Invalid file extension '{file_extension}'") - matching_extensions = glob.glob(f'../tmp/*.{file_extension}') + matching_extensions = glob.glob(f"../tmp/*.{file_extension}") if not matching_extensions: logger.error(f"No files with the extension {file_extension} found") @@ -39,23 +41,25 @@ class Command(BaseCommand): filename = os.path.basename(src_file_path) do_command = True exit_status: int - - desired_file_path = f'{directory}/{filename}' + + desired_file_path = f"{directory}/{filename}" if os.path.exists(desired_file_path): - replace = input(f'{desired_file_path} already exists. Do you want to replace it? (y/n) ') - if replace.lower() != 'y': + replace = input( + f"{desired_file_path} already exists. Do you want to replace it? (y/n) " + ) + if replace.lower() != "y": do_command = False - + if do_command: copy_from = f"../tmp/{filename}" self.cat(copy_from, desired_file_path) - exit_status = os.system(f'cat ../tmp/{filename} > {desired_file_path}') + exit_status = os.system(f"cat ../tmp/{filename} > {desired_file_path}") if exit_status == 0: logger.info(f"Successfully copied {filename}") else: logger.info(f"Failed to copy {filename}") - + def cat(self, copy_from, copy_to): - exit_status = os.system(f'cat {copy_from} > {copy_to}') - return exit_status \ No newline at end of file + exit_status = os.system(f"cat {copy_from} > {copy_to}") + return exit_status diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index 866508fb1..41e9856ce 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -11,40 +11,98 @@ from django.core.management import BaseCommand from registrar.models.transition_domain import TransitionDomain from .utility.extra_transition_domain import ExtraTransitionDomain -from .utility.epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, EnumFilenames +from .utility.epp_data_containers import ( + AgencyAdhoc, + DomainAdditionalData, + DomainTypeAdhoc, + OrganizationAdhoc, + EnumFilenames, +) logger = logging.getLogger(__name__) +class LogCode(Enum): + ERROR = 1 + WARNING = 2 + INFO = 3 + DEBUG = 4 + + +class FileTransitionLog: + def __init__(self): + self.logs = { + EnumFilenames.DOMAIN_ADHOC: [], + EnumFilenames.AGENCY_ADHOC: [], + EnumFilenames.ORGANIZATION_ADHOC: [], + EnumFilenames.DOMAIN_ADDITIONAL: [], + } + + class LogItem: + def __init__(self, file_type, code, message): + self.file_type = file_type + self.code = code + self.message = message + + def add_log(self, file_type, code, message): + self.logs[file_type] = self.LogItem(file_type, code, message) + + def add_log(self, log: LogItem): + self.logs.append(log) + + def create_log_item(self, file_type, code, message, add_to_list=True): + """Creates and returns an LogItem object. + + add_to_list: bool -> If enabled, add it to the logs array. + """ + log = self.LogItem(file_type, code, message) + if not add_to_list: + return log + else: + self.logs[file_type] = log + return log + + def display_logs(self, file_type): + for log in self.logs.get(file_type): + match log.code: + case LogCode.ERROR: + logger.error(log.message) + case LogCode.WARNING: + logger.warning(log.message) + case LogCode.INFO: + logger.info(log.message) + case LogCode.DEBUG: + logger.debug(log.message) + + class Command(BaseCommand): help = "" filenames = EnumFilenames + parse_logs = FileTransitionLog() def add_arguments(self, parser): """Add filename arguments.""" parser.add_argument( - "--directory", - default="migrationdata", - help="Desired directory" + "--directory", default="migrationdata", help="Desired directory" ) parser.add_argument( "--agency_adhoc_filename", - default=self.filenames.AGENCY_ADHOC[1], + default=EnumFilenames.AGENCY_ADHOC[1], help="Defines the filename for agency adhocs", ) parser.add_argument( "--domain_additional_filename", - default=self.filenames.DOMAIN_ADDITIONAL[1], + default=EnumFilenames.DOMAIN_ADDITIONAL[1], help="Defines the filename for additional domain data", ) parser.add_argument( "--domain_adhoc_filename", - default=self.filenames.DOMAIN_ADHOC[1], + default=EnumFilenames.DOMAIN_ADHOC[1], help="Defines the filename for domain type adhocs", ) parser.add_argument( "--organization_adhoc_filename", - default=self.filenames.ORGANIZATION_ADHOC[1], + default=EnumFilenames.ORGANIZATION_ADHOC[1], help="Defines the filename for domain type adhocs", ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -52,41 +110,259 @@ class Command(BaseCommand): def handle(self, **options): try: self.domain_object = ExtraTransitionDomain( - agency_adhoc_filename=options['agency_adhoc_filename'], - domain_additional_filename=options['domain_additional_filename'], - domain_adhoc_filename=options['domain_adhoc_filename'], - organization_adhoc_filename=options['organization_adhoc_filename'], - directory=options['directory'], - seperator=options['sep'] + agency_adhoc_filename=options["agency_adhoc_filename"], + domain_additional_filename=options["domain_additional_filename"], + domain_adhoc_filename=options["domain_adhoc_filename"], + organization_adhoc_filename=options["organization_adhoc_filename"], + directory=options["directory"], + seperator=options["sep"], ) self.domain_object.parse_all_files() except Exception as err: logger.error(f"Could not load additional data. Error: {err}") else: - for transition_domain in TransitionDomain.objects.all(): - transition_domain.organization_type + all_transition_domains = TransitionDomain.objects.all() + if not all_transition_domains.exists(): + raise Exception("No TransitionDomain objects exist.") - def get_organization_adhoc(self, desired_id): + for transition_domain in all_transition_domains: + domain_name = transition_domain.domain_name + updated_transition_domain = transition_domain + + # STEP 1: Parse domain type data + updated_transition_domain = self.parse_domain_type_data( + domain_name, transition_domain + ) + self.parse_logs(EnumFilenames.DOMAIN_ADHOC) + + # STEP 2: Parse agency data - TODO + updated_transition_domain = self.parse_agency_data( + domain_name, transition_domain + ) + self.parse_logs(EnumFilenames.AGENCY_ADHOC) + + # STEP 3: Parse organization data + updated_transition_domain = self.parse_org_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.ORGANIZATION_ADHOC) + + # STEP 4: Parse expiration data - TODO + updated_transition_domain = self.parse_expiration_data( + domain_name, transition_domain + ) + # self.parse_logs(EnumFilenames.EXPIRATION_DATA) + + updated_transition_domain.save() + + # TODO - Implement once Niki gets her ticket in + def parse_expiration_data(self, domain_name, transition_domain): + return transition_domain + + # TODO - Implement once Niki gets her ticket in + def parse_agency_data(self, domain_name, transition_domain): + """ + + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + info = self.get_domain_type_info(domain_name) + if info is None: + self.parse_logs.create_log_item( + EnumFilenames.AGENCY_ADHOC, + LogCode.INFO, + f"Could not add agency_data on {domain_name}, no data exists." + ) + return transition_domain + + agency_exists = ( + transition_domain.agency_name is not None + and transition_domain.agency_name.strip() != "" + ) + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.AGENCY_ADHOC, + "agency_name", + transition_domain.agency_name, + domain_name, + agency_exists + ) + """ + return transition_domain + + def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain): + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + info = self.get_domain_type_info(domain_name) + if info is None: + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.INFO, + f"Could not add domain_type on {domain_name}, no data exists.", + ) + return transition_domain + + # This data is stored as follows: FEDERAL - Judicial + # For all other records, it is stored as so: Interstate + # We can infer if it is federal or not based on this fact. + domain_type = info.domaintype.split("-") + if domain_type.count != 1 or domain_type.count != 2: + raise ValueError("Found invalid data in DOMAIN_ADHOC") + + # Then, just grab the agency type. + new_federal_agency = domain_type[0].strip() + + # Check if this domain_type is active or not. + # If not, we don't want to add this. + if not info.active.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", + ) + return transition_domain + + # Are we updating data that already exists, + # or are we adding new data in its place? + federal_agency_exists = ( + transition_domain.federal_agency is not None + and transition_domain.federal_agency.strip() != "" + ) + federal_type_exists = ( + transition_domain.federal_type is not None + and transition_domain.federal_type.strip() != "" + ) + + # If we get two records, then we know it is federal. + is_federal = domain_type.count() == 2 + if is_federal: + new_federal_type = domain_type[1].strip() + transition_domain.federal_agency = new_federal_agency + transition_domain.federal_type = new_federal_type + else: + transition_domain.federal_agency = new_federal_agency + transition_domain.federal_type = None + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.DOMAIN_ADHOC, + "federal_agency", + transition_domain.federal_agency, + domain_name, + federal_agency_exists, + ) + + self._add_or_change_message( + EnumFilenames.DOMAIN_ADHOC, + "federal_type", + transition_domain.federal_type, + domain_name, + federal_type_exists, + ) + + return transition_domain + + def parse_org_data(self, domain_name, transition_domain: TransitionDomain): + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + org_info = self.get_org_info(domain_name) + if org_info is None: + self.parse_logs.create_log_item( + EnumFilenames.ORGANIZATION_ADHOC, + LogCode.INFO, + f"Could not add organization_type on {domain_name}, no data exists.", + ) + return transition_domain + + desired_property_exists = ( + transition_domain.organization_type is not None + and transition_domain.organization_type.strip() != "" + ) + + transition_domain.organization_type = org_info.orgname + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.ORGANIZATION_ADHOC, + "organization_type", + transition_domain.organization_type, + domain_name, + desired_property_exists, + ) + + return transition_domain + + def _add_or_change_message( + self, file_type, var_name, changed_value, domain_name, is_update=False + ): + """Creates a log instance when a property + is successfully changed on a given TransitionDomain.""" + if not is_update: + self.parse_logs.create_log_item( + file_type, + LogCode.DEBUG, + f"Added {file_type} as '{var_name}' on {domain_name}", + ) + else: + self.parse_logs.create_log_item( + file_type, + LogCode.INFO, + f"Updated existing {var_name} to '{changed_value}' on {domain_name}", + ) + + def get_org_info(self, domain_name) -> OrganizationAdhoc: + domain_info = self.get_domain_data(domain_name) + org_id = domain_info.orgid + return self.get_organization_adhoc(org_id) + + def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: + domain_info = self.get_domain_data(domain_name) + type_id = domain_info.domaintypeid + return self.get_domain_adhoc(type_id) + + def get_agency_info(self, domain_name): + # domain_info = self.get_domain_data(domain_name) + # type_id = domain_info.authorityid + # return self.get_domain_adhoc(type_id) + raise + + def get_domain_data(self, desired_id) -> DomainAdditionalData: + return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) + + def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: """Grabs adhoc information for organizations. Returns an organization - dictionary - - returns: - { - " + dictionary. + + returns: + { + "org_id_1": OrganizationAdhoc, + "org_id_2: OrganizationAdhoc, + ... } """ - return self.get_object_by_id(self.filenames.ORGANIZATION_ADHOC, desired_id) - + return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) + def get_domain_adhoc(self, desired_id): """""" - return self.get_object_by_id(self.filenames.DOMAIN_ADHOC, desired_id) - + return self.get_object_by_id(EnumFilenames.DOMAIN_ADHOC, desired_id) + def get_agency_adhoc(self, desired_id): """""" - return self.get_object_by_id(self.filenames.AGENCY_ADHOC, desired_id) - + return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) + def get_object_by_id(self, file_type: EnumFilenames, desired_id): """""" desired_type = self.domain_object.csv_data.get(file_type) - obj = desired_type.get(desired_id) + if desired_type is not None: + obj = desired_type.get(desired_id) + else: + self.parse_logs.create_log_item( + file_type, LogCode.ERROR, f"Id {desired_id} does not exist" + ) return obj diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 624418fe9..4fb5b2e6e 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -88,9 +88,6 @@ class Command(BaseCommand): parser.add_argument( "domain_statuses_filename", help="Data file with domain status information" ) - parser.add_argument( - "--loadExtraData", default=True, help="Determines if additional metadata should be applied" - ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -326,9 +323,6 @@ class Command(BaseCommand): # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) - - if load_extra_data: - # STEP 1: # Create mapping of domain name -> status diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index bc9efa9df..5910d2bbf 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -17,10 +17,13 @@ from registrar.models.domain_information import DomainInformation from registrar.management.commands.utility.terminal_helper import TerminalColors from registrar.management.commands.utility.terminal_helper import TerminalHelper -from registrar.management.commands.load_transition_domain import Command as load_transition_domain_command +from registrar.management.commands.load_transition_domain import ( + Command as load_transition_domain_command, +) logger = logging.getLogger(__name__) + class Command(BaseCommand): help = """ """ @@ -31,19 +34,23 @@ class Command(BaseCommand): A boolean (default to true), which activates additional print statements """ - parser.add_argument("--runLoaders", + parser.add_argument( + "--runLoaders", help="Runs all scripts (in sequence) for transition domain migrations", - action=argparse.BooleanOptionalAction) - - parser.add_argument("--triggerLogins", + action=argparse.BooleanOptionalAction, + ) + + parser.add_argument( + "--triggerLogins", help="Simulates a user login for each user in domain invitation", - action=argparse.BooleanOptionalAction) + action=argparse.BooleanOptionalAction, + ) # The following file arguments have default values for running in the sandbox parser.add_argument( "--loaderDirectory", default="migrationData", - help="The location of the files used for load_transition_domain migration script" + help="The location of the files used for load_transition_domain migration script", ) parser.add_argument( "--loaderFilenames", @@ -55,7 +62,7 @@ class Command(BaseCommand): where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""" + - domain_statuses_filename is the Data file with domain status information""", ) # parser.add_argument( @@ -74,7 +81,9 @@ class Command(BaseCommand): # help="Data file with domain status information" # ) - parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") + parser.add_argument( + "--sep", default="|", help="Delimiter character for the loader files" + ) parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -88,9 +97,7 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - def print_debug_mode_statements( - self, debug_on: bool - ): + def print_debug_mode_statements(self, debug_on: bool): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" self.print_debug( @@ -119,8 +126,8 @@ class Command(BaseCommand): """ ) - #TODO: would filteredRelation be faster? - for transition_domain in TransitionDomain.objects.all():# DEBUG: + # TODO: would filteredRelation be faster? + for transition_domain in TransitionDomain.objects.all(): # DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username @@ -137,10 +144,14 @@ class Command(BaseCommand): # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + matching_domain_informations = DomainInformation.objects.filter( + domain__name=transition_domain_name + ) # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), - domain__name=transition_domain_name) + matching_domain_invitations = DomainInvitation.objects.filter( + email=transition_domain_email.lower(), + domain__name=transition_domain_name, + ) if len(matching_domains) == 0: missing_domains.append(transition_domain_name) @@ -157,10 +168,16 @@ class Command(BaseCommand): total_missing_domain_invitations = len(missing_domain_invites) missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) - duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) - missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) - missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) - + duplicate_domains_as_string = "{}".format( + ", ".join(map(str, duplicate_domains)) + ) + missing_domain_informations_as_string = "{}".format( + ", ".join(map(str, missing_domain_informations)) + ) + missing_domain_invites_as_string = "{}".format( + ", ".join(map(str, missing_domain_invites)) + ) + logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED ANALYSIS =============== @@ -183,20 +200,26 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ ) - - def run_load_transition_domain_script(self, - file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - debug_max_entries_to_parse): + + def run_load_transition_domain_script( + self, + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse, + ): load_transition_domain_command_string = "./manage.py load_transition_domain " - load_transition_domain_command_string += file_location+domain_contacts_filename + " " - load_transition_domain_command_string += file_location+contacts_filename + " " - load_transition_domain_command_string += file_location+domain_statuses_filename + " " + load_transition_domain_command_string += ( + file_location + domain_contacts_filename + " " + ) + load_transition_domain_command_string += file_location + contacts_filename + " " + load_transition_domain_command_string += ( + file_location + domain_statuses_filename + " " + ) if sep is not None and sep != "|": load_transition_domain_command_string += f"--sep {sep} " @@ -208,7 +231,9 @@ class Command(BaseCommand): load_transition_domain_command_string += "--debug " if debug_max_entries_to_parse > 0: - load_transition_domain_command_string += f"--limitParse {debug_max_entries_to_parse} " + load_transition_domain_command_string += ( + f"--limitParse {debug_max_entries_to_parse} " + ) proceed_load_transition_domain = TerminalHelper.query_yes_no( f"""{TerminalColors.OKCYAN} @@ -224,18 +249,19 @@ class Command(BaseCommand): if not proceed_load_transition_domain: return - logger.info(f"""{TerminalColors.OKCYAN} + logger.info( + f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== - {TerminalColors.ENDC}""") + {TerminalColors.ENDC}""" + ) os.system(f"{load_transition_domain_command_string}") - + def run_transfer_script(self, debug_on): command_string = "./manage.py transfer_transition_domains_to_domains " if debug_on: command_string += "--debug " - proceed_load_transition_domain = TerminalHelper.query_yes_no( f"""{TerminalColors.OKCYAN} ===================================================== @@ -250,18 +276,20 @@ class Command(BaseCommand): if not proceed_load_transition_domain: return - logger.info(f"""{TerminalColors.OKCYAN} + logger.info( + f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== - {TerminalColors.ENDC}""") + {TerminalColors.ENDC}""" + ) os.system(f"{command_string}") - def run_migration_scripts(self, - options): - file_location = options.get("loaderDirectory")+"/" + def run_migration_scripts(self, options): + file_location = options.get("loaderDirectory") + "/" filenames = options.get("loaderFilenames").split() if len(filenames) < 3: filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" + logger.info( + f""" {TerminalColors.FAIL} --loaderFilenames expected 3 filenames to follow it, but only {len(filenames)} were given: @@ -270,7 +298,8 @@ class Command(BaseCommand): PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN ============= TERMINATING ============= {TerminalColors.ENDC} - """) + """ + ) return domain_contacts_filename = filenames[0] contacts_filename = filenames[1] @@ -295,16 +324,18 @@ class Command(BaseCommand): if not files_are_correct: # prompt the user to provide correct file inputs - logger.info(f""" + logger.info( + f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: EXAMPLE: docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - """) + """ + ) return - + # Get --sep argument sep = options.get("sep") @@ -319,53 +350,65 @@ class Command(BaseCommand): options.get("limitParse") ) # set to 0 to parse all entries - self.run_load_transition_domain_script(file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - debug_max_entries_to_parse) + self.run_load_transition_domain_script( + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse, + ) self.run_transfer_script(debug_on) def simulate_user_logins(self, debug_on): - logger.info(f"""{TerminalColors.OKCYAN} + logger.info( + f"""{TerminalColors.OKCYAN} ================== SIMULATING LOGINS ================== {TerminalColors.ENDC} - """) + """ + ) for invite in DomainInvitation.objects.all(): - #DEBUG: - TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") - # get a user with this email address + # DEBUG: + TerminalHelper.print_debug( + debug_on, + f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""", + ) + # get a user with this email address User = get_user_model() try: user = User.objects.get(email=invite.email) - #DEBUG: - TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") + # DEBUG: + TerminalHelper.print_debug( + debug_on, + f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""", + ) Client.force_login(user) except User.DoesNotExist: - #TODO: how should we handle this? - logger.warn(f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""") + # TODO: how should we handle this? + logger.warn( + f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""" + ) def handle( self, **options, ): """ - Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. - + Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Produces the following report (printed to the terminal): #1 - Print any domains that exist in the transition_domain table but not in their corresponding domain, domain information or domain invitation tables. #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are + #3- Check for duplicate entries in domain or + domain_information tables and print which are duplicates and in which tables (ONLY RUNS with full script option) @@ -374,10 +417,10 @@ class Command(BaseCommand): on django admin for an analyst OPTIONS: - -- (run all other scripts: + -- (run all other scripts: 1 - imports for trans domains 2 - transfer to domain & domain invitation - 3 - send domain invite) + 3 - send domain invite) ** Triggers table reset ** """ @@ -394,12 +437,12 @@ class Command(BaseCommand): if run_loaders_on: self.run_migration_scripts(options) prompt_continuation_of_analysis = True - + # Simulate user login for each user in domain invitation if sepcified by user if simulate_user_login_enabled: self.simulate_user_logins(debug_on) prompt_continuation_of_analysis = True - + analyze_tables = True if prompt_continuation_of_analysis: analyze_tables = TerminalHelper.query_yes_no( @@ -410,4 +453,4 @@ class Command(BaseCommand): # Analyze tables for corrupt data... if analyze_tables: - self.compare_tables(debug_on) \ No newline at end of file + self.compare_tables(debug_on) diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 2d006c42b..3fe60da2b 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -2,18 +2,21 @@ from dataclasses import dataclass from enum import Enum from typing import Optional + @dataclass -class AgencyAdhoc(): - """Defines the structure given in the given AGENCY_ADHOC file""" +class AgencyAdhoc: + """Defines the structure given in the AGENCY_ADHOC file""" + agencyid: Optional[int] = None agencyname: Optional[str] = None - active: Optional[bool] = None - isfederal: Optional[bool] = None + active: Optional[str] = None + isfederal: Optional[str] = None @dataclass -class DomainAdditionalData(): - """Defines the structure given in the given DOMAIN_ADDITIONAL file""" +class DomainAdditionalData: + """Defines the structure given in the DOMAIN_ADDITIONAL file""" + domainname: Optional[str] = None domaintypeid: Optional[int] = None authorityid: Optional[int] = None @@ -22,17 +25,21 @@ class DomainAdditionalData(): dnsseckeymonitor: Optional[str] = None domainpurpose: Optional[str] = None + @dataclass -class DomainTypeAdhoc(): - """Defines the structure given in the given DOMAIN_ADHOC file""" +class DomainTypeAdhoc: + """Defines the structure given in the DOMAIN_ADHOC file""" + domaintypeid: Optional[int] = None domaintype: Optional[str] = None code: Optional[str] = None - active: Optional[bool] = None + active: Optional[str] = None + @dataclass -class OrganizationAdhoc(): - """Defines the structure given in the given ORGANIZATION_ADHOC file""" +class OrganizationAdhoc: + """Defines the structure given in the ORGANIZATION_ADHOC file""" + orgid: Optional[int] = None orgname: Optional[str] = None orgstreet: Optional[str] = None @@ -41,12 +48,17 @@ class OrganizationAdhoc(): orgzip: Optional[str] = None orgcountrycode: Optional[str] = None + class EnumFilenames(Enum): - """Returns a tuple mapping for (filetype, default_file_name). - + """Returns a tuple mapping for (filetype, default_file_name). + For instance, AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") """ + AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") - DOMAIN_ADDITIONAL = ("domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt") + DOMAIN_ADDITIONAL = ( + "domain_additional", + "domainadditionaldatalink.adhoc.dotgov.txt", + ) DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") - ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") \ No newline at end of file + ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index a54540960..2010fe563 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -7,20 +7,26 @@ import logging import os from typing import List -from epp_data_containers import AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, EnumFilenames +from epp_data_containers import ( + AgencyAdhoc, + DomainAdditionalData, + DomainTypeAdhoc, + OrganizationAdhoc, + EnumFilenames, +) logger = logging.getLogger(__name__) @dataclass -class PatternMap(): +class PatternMap: """Helper class that holds data and metadata about a requested file. - + filename: str -> The desired filename to target. If no filename is given, it is assumed that you are passing in a filename pattern and it will look for a filename that matches the given postfix you pass in. - regex: re.Pattern -> Defines what regex you want to use when inferring + regex: re.Pattern -> Defines what regex you want to use when inferring filenames. If none, no matching occurs. data_type: type -> Metadata about the desired type for data. @@ -31,7 +37,15 @@ class PatternMap(): to cross-reference. """ - def __init__(self, filename: str, regex: re.Pattern, data_type: type, id_field: str, data: dict = {}): + + def __init__( + self, + filename: str, + regex: re.Pattern, + data_type: type, + id_field: str, + data: dict = {}, + ): self.regex = regex self.data_type = data_type self.id_field = id_field @@ -41,16 +55,15 @@ class PatternMap(): _infer = self._infer_filename(self.regex, filename) self.filename = _infer[0] self.could_infer = _infer[1] - def _infer_filename(self, regex: re.Pattern, default_file_name): if not isinstance(regex, re.Pattern): return (self.filename, False) - + match = regex.match(self.filename) if not match: return (self.filename, False) - + date = match.group(1) filename_without_date = match.group(2) @@ -61,17 +74,19 @@ class PatternMap(): full_filename = date + filename_without_date return (full_filename, can_infer) -class ExtraTransitionDomain(): + +class ExtraTransitionDomain: filenames = EnumFilenames - strip_date_regex = re.compile(r'\d+\.(.+)') - - def __init__(self, + strip_date_regex = re.compile(r"\d+\.(.+)") + + def __init__( + self, agency_adhoc_filename=filenames.AGENCY_ADHOC[1], domain_additional_filename=filenames.DOMAIN_ADDITIONAL[1], domain_adhoc_filename=filenames.DOMAIN_ADHOC[1], organization_adhoc_filename=filenames.ORGANIZATION_ADHOC[1], directory="migrationdata", - seperator="|" + seperator="|", ): self.directory = directory self.seperator = seperator @@ -81,18 +96,34 @@ class ExtraTransitionDomain(): self.csv_data = { # (filename, default_url): metadata about the desired file - self.filenames.AGENCY_ADHOC: PatternMap(agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc, "agencyid"), - self.filenames.DOMAIN_ADDITIONAL: PatternMap(domain_additional_filename, self.strip_date_regex, DomainAdditionalData, "domainname"), - self.filenames.DOMAIN_ADHOC: PatternMap(domain_adhoc_filename, self.strip_date_regex, DomainTypeAdhoc, "domaintypeid"), - self.filenames.ORGANIZATION_ADHOC: PatternMap(organization_adhoc_filename, self.strip_date_regex, OrganizationAdhoc, "orgid") + self.filenames.AGENCY_ADHOC: PatternMap( + agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc, "agencyid" + ), + self.filenames.DOMAIN_ADDITIONAL: PatternMap( + domain_additional_filename, + self.strip_date_regex, + DomainAdditionalData, + "domainname", + ), + self.filenames.DOMAIN_ADHOC: PatternMap( + domain_adhoc_filename, + self.strip_date_regex, + DomainTypeAdhoc, + "domaintypeid", + ), + self.filenames.ORGANIZATION_ADHOC: PatternMap( + organization_adhoc_filename, + self.strip_date_regex, + OrganizationAdhoc, + "orgid", + ), } - - def parse_all_files(self, overwrite_existing_data = True): + def parse_all_files(self, overwrite_existing_data=True): """Clears all preexisting data then parses each related CSV file. - - overwrite_existing_data: bool -> Determines if we should clear - csv_data.data if it already exists + + overwrite_existing_data: bool -> Determines if we should clear + csv_data.data if it already exists """ self.clear_csv_data() for item in self.csv_data: @@ -101,16 +132,15 @@ class ExtraTransitionDomain(): if filename in self.all_files_set: file_type.data = self._read_csv_file( - self.all_files_set[filename], - self.seperator, + self.all_files_set[filename], + self.seperator, file_type.data_type, - file_type.id_field + file_type.id_field, ) else: # Log if we can't find the desired file logger.error(f"Could not find file: {filename}") - def clear_csv_data(self): for item in self.csv_data: file_type: PatternMap = item.value @@ -120,4 +150,3 @@ class ExtraTransitionDomain(): with open(file, "r", encoding="utf-8") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) return {row[id_field]: dataclass_type(**row) for row in reader} - diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index ec7580e21..ad20ad4ca 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -2,6 +2,7 @@ import logging logger = logging.getLogger(__name__) + class TerminalColors: """Colors for terminal outputs (makes reading the logs WAY easier)""" @@ -17,8 +18,8 @@ class TerminalColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" -class TerminalHelper: +class TerminalHelper: def query_yes_no(question: str, default="yes") -> bool: """Ask a yes/no question via raw_input() and return their answer. @@ -56,4 +57,4 @@ class TerminalHelper: terminal if print_condition is TRUE""" # DEBUG: if print_condition: - logger.info(print_statement) \ No newline at end of file + logger.info(print_statement) diff --git a/src/registrar/migrations/0044_transitiondomain_organization_name.py b/src/registrar/migrations/0044_transitiondomain_organization_name.py deleted file mode 100644 index 07c6ece82..000000000 --- a/src/registrar/migrations/0044_transitiondomain_organization_name.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.6 on 2023-10-27 14:21 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("registrar", "0043_transitiondomain_federal_agency_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="transitiondomain", - name="organization_name", - field=models.TextField( - blank=True, help_text="Organization name", null=True - ), - ), - ] diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 6670b7bbd..d95d8e441 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -48,11 +48,6 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Type of organization", ) - organization_name = models.TextField( - null=True, - blank=True, - help_text="Organization name", - ) federal_type = models.TextField( max_length=50, null=True, From a74b9f4c3c37ea587b7f215e727fc753a7dccd6c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:44:25 -0600 Subject: [PATCH 08/88] Parsing agency, documentation --- src/migrationdata/README.md | 4 +- .../commands/load_extra_transition_domain.py | 202 ++++++++++++------ .../commands/master_domain_migrations.py | 23 +- .../commands/utility/epp_data_containers.py | 29 ++- .../utility/extra_transition_domain.py | 33 ++- src/registrar/models/transition_domain.py | 6 + 6 files changed, 210 insertions(+), 87 deletions(-) diff --git a/src/migrationdata/README.md b/src/migrationdata/README.md index 81190ee3f..585624bdb 100644 --- a/src/migrationdata/README.md +++ b/src/migrationdata/README.md @@ -1,8 +1,8 @@ ## Purpose -Use this folder for storing files for the migration process. Should otherwise be empty on local dev environments unless necessary. This folder must exist due to the nature of how data is stored on cloud.gov and the nature of the data we want to send. +Use this folder for storing files for the migration process. Should otherwise be empty on local dev environments unless necessary. This folder must exist due to the nature of how data is stored on cloud.gov and the nature of the data we typically want to send. ## How do I migrate registrar data? This process is detailed in [data_migration.md](../../docs/operations/data_migration.md) ## What kind of files can I store here? -The intent is for PII data or otherwise, but this can exist in any format. Do note that the data contained in this file will be temporary, so after the app is restaged it will lose it. This is ideal for migration files as they write to our DB, but not for something you need to permanently hold onto. \ No newline at end of file +The intent is for PII data or otherwise, but this can exist in any format. Do note that the data contained in this file will be temporary, so after the app is restaged it will lose it (as long as nothing is committed). This is ideal for migration files as they write to our DB, but not for something you need to permanently hold onto. \ No newline at end of file diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index 41e9856ce..f219081e1 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -13,6 +13,7 @@ from registrar.models.transition_domain import TransitionDomain from .utility.extra_transition_domain import ExtraTransitionDomain from .utility.epp_data_containers import ( AgencyAdhoc, + AuthorityAdhoc, DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, @@ -30,6 +31,17 @@ class LogCode(Enum): class FileTransitionLog: + """Container for storing event logs. Used to lessen + the complexity of storing multiple logs across multiple + variables. + + self.logs: dict -> { + EnumFilenames.DOMAIN_ADHOC: List[LogItem], + EnumFilenames.AGENCY_ADHOC: List[LogItem], + EnumFilenames.ORGANIZATION_ADHOC: List[LogItem], + EnumFilenames.DOMAIN_ADDITIONAL: List[LogItem], + } + """ def __init__(self): self.logs = { EnumFilenames.DOMAIN_ADHOC: [], @@ -39,16 +51,24 @@ class FileTransitionLog: } class LogItem: + """Used for storing data about logger information. + Intended for use in""" def __init__(self, file_type, code, message): self.file_type = file_type self.code = code self.message = message def add_log(self, file_type, code, message): - self.logs[file_type] = self.LogItem(file_type, code, message) + """Adds a log item to self.logs - def add_log(self, log: LogItem): - self.logs.append(log) + file_type -> Which array to add to, + ex. EnumFilenames.DOMAIN_ADHOC + + code -> Log severity or other metadata, ex. LogCode.ERROR + + message -> Message to display + """ + self.logs[file_type] = self.LogItem(file_type, code, message) def create_log_item(self, file_type, code, message, add_to_list=True): """Creates and returns an LogItem object. @@ -63,6 +83,9 @@ class FileTransitionLog: return log def display_logs(self, file_type): + """Displays all logs in the given file_type in EnumFilenames. + Will log with the correct severity depending on code. + """ for log in self.logs.get(file_type): match log.code: case LogCode.ERROR: @@ -129,24 +152,24 @@ class Command(BaseCommand): domain_name = transition_domain.domain_name updated_transition_domain = transition_domain - # STEP 1: Parse domain type data - updated_transition_domain = self.parse_domain_type_data( - domain_name, transition_domain - ) - self.parse_logs(EnumFilenames.DOMAIN_ADHOC) - - # STEP 2: Parse agency data - TODO - updated_transition_domain = self.parse_agency_data( - domain_name, transition_domain - ) - self.parse_logs(EnumFilenames.AGENCY_ADHOC) - - # STEP 3: Parse organization data + # STEP 1: Parse organization data updated_transition_domain = self.parse_org_data( domain_name, transition_domain ) self.parse_logs.display_logs(EnumFilenames.ORGANIZATION_ADHOC) + # STEP 2: Parse domain type data + updated_transition_domain = self.parse_domain_type_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.DOMAIN_ADHOC) + + # STEP 3: Parse agency data - TODO + updated_transition_domain = self.parse_agency_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.AGENCY_ADHOC) + # STEP 4: Parse expiration data - TODO updated_transition_domain = self.parse_expiration_data( domain_name, transition_domain @@ -159,40 +182,59 @@ class Command(BaseCommand): def parse_expiration_data(self, domain_name, transition_domain): return transition_domain - # TODO - Implement once Niki gets her ticket in - def parse_agency_data(self, domain_name, transition_domain): - """ - + def parse_agency_data(self, domain_name, transition_domain) -> TransitionDomain: if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") - info = self.get_domain_type_info(domain_name) + info = self.get_agency_info(domain_name) if info is None: self.parse_logs.create_log_item( EnumFilenames.AGENCY_ADHOC, LogCode.INFO, - f"Could not add agency_data on {domain_name}, no data exists." + f"Could not add federal_agency on {domain_name}, no data exists." ) return transition_domain agency_exists = ( - transition_domain.agency_name is not None - and transition_domain.agency_name.strip() != "" + transition_domain.federal_agency is not None + and transition_domain.federal_agency.strip() != "" ) + if not info.active.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add inactive agency {info.agencyname} on {domain_name}", + ) + return transition_domain + + if not info.isfederal.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add non-federal agency {info.agencyname} on {domain_name}", + ) + return transition_domain + + transition_domain.federal_agency = info.agencyname + # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.AGENCY_ADHOC, - "agency_name", - transition_domain.agency_name, + "federal_agency", + transition_domain.federal_agency, domain_name, agency_exists ) - """ + return transition_domain - def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain): + def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: + """Parses the DomainType file. + This file has definitions for organization_type and federal_agency. + Logs if + """ if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") @@ -212,8 +254,8 @@ class Command(BaseCommand): if domain_type.count != 1 or domain_type.count != 2: raise ValueError("Found invalid data in DOMAIN_ADHOC") - # Then, just grab the agency type. - new_federal_agency = domain_type[0].strip() + # Then, just grab the organization type. + new_organization_type = domain_type[0].strip() # Check if this domain_type is active or not. # If not, we don't want to add this. @@ -228,7 +270,7 @@ class Command(BaseCommand): # Are we updating data that already exists, # or are we adding new data in its place? federal_agency_exists = ( - transition_domain.federal_agency is not None + transition_domain.organization_type is not None and transition_domain.federal_agency.strip() != "" ) federal_type_exists = ( @@ -237,13 +279,14 @@ class Command(BaseCommand): ) # If we get two records, then we know it is federal. + # needs to be lowercase for federal type is_federal = domain_type.count() == 2 if is_federal: new_federal_type = domain_type[1].strip() - transition_domain.federal_agency = new_federal_agency + transition_domain.organization_type = new_organization_type transition_domain.federal_type = new_federal_type else: - transition_domain.federal_agency = new_federal_agency + transition_domain.organization_type = new_organization_type transition_domain.federal_type = None # Logs if we either added to this property, @@ -266,7 +309,7 @@ class Command(BaseCommand): return transition_domain - def parse_org_data(self, domain_name, transition_domain: TransitionDomain): + def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") @@ -275,23 +318,23 @@ class Command(BaseCommand): self.parse_logs.create_log_item( EnumFilenames.ORGANIZATION_ADHOC, LogCode.INFO, - f"Could not add organization_type on {domain_name}, no data exists.", + f"Could not add organization_name on {domain_name}, no data exists.", ) return transition_domain desired_property_exists = ( - transition_domain.organization_type is not None - and transition_domain.organization_type.strip() != "" + transition_domain.organization_name is not None + and transition_domain.organization_name.strip() != "" ) - transition_domain.organization_type = org_info.orgname + transition_domain.organization_name = org_info.orgname # Logs if we either added to this property, # or modified it. self._add_or_change_message( EnumFilenames.ORGANIZATION_ADHOC, - "organization_type", - transition_domain.organization_type, + "organization_name", + transition_domain.organization_name, domain_name, desired_property_exists, ) @@ -316,6 +359,7 @@ class Command(BaseCommand): f"Updated existing {var_name} to '{changed_value}' on {domain_name}", ) + # Property getters, i.e. orgid or domaintypeid def get_org_info(self, domain_name) -> OrganizationAdhoc: domain_info = self.get_domain_data(domain_name) org_id = domain_info.orgid @@ -326,43 +370,81 @@ class Command(BaseCommand): type_id = domain_info.domaintypeid return self.get_domain_adhoc(type_id) - def get_agency_info(self, domain_name): - # domain_info = self.get_domain_data(domain_name) - # type_id = domain_info.authorityid - # return self.get_domain_adhoc(type_id) - raise + def get_agency_info(self, domain_name) -> AgencyAdhoc: + domain_info = self.get_domain_data(domain_name) + type_id = domain_info.orgid + return self.get_domain_adhoc(type_id) + + def get_authority_info(self, domain_name): + domain_info = self.get_domain_data(domain_name) + type_id = domain_info.authorityid + return self.get_authority_adhoc(type_id) + # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc def get_domain_data(self, desired_id) -> DomainAdditionalData: return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: """Grabs adhoc information for organizations. Returns an organization - dictionary. - - returns: - { - "org_id_1": OrganizationAdhoc, - "org_id_2: OrganizationAdhoc, - ... - } + adhoc object. """ return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) - def get_domain_adhoc(self, desired_id): + def get_domain_adhoc(self, desired_id) -> DomainTypeAdhoc: """""" return self.get_object_by_id(EnumFilenames.DOMAIN_ADHOC, desired_id) - def get_agency_adhoc(self, desired_id): + def get_agency_adhoc(self, desired_id) -> AgencyAdhoc: """""" return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) + + def get_authority_adhoc(self, desired_id) -> AuthorityAdhoc: + """""" + return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) def get_object_by_id(self, file_type: EnumFilenames, desired_id): - """""" - desired_type = self.domain_object.csv_data.get(file_type) - if desired_type is not None: - obj = desired_type.get(desired_id) - else: + """Returns a field in a dictionary based off the type and id. + + vars: + file_type: (constant) EnumFilenames -> Which data file to target. + An example would be `EnumFilenames.DOMAIN_ADHOC`. + + desired_id: str -> Which id you want to search on. + An example would be `"12"` or `"igorville.gov"` + + Explanation: + Each data file has an associated type (file_type) for tracking purposes. + + Each file_type is a dictionary which + contains a dictionary of row[id_field]: object. + + In practice, this would look like: + + EnumFilenames.AUTHORITY_ADHOC: { + "1": AuthorityAdhoc(...), + "2": AuthorityAdhoc(...), + ... + } + + desired_id will then specify which id to grab. If we wanted "1", + then this function will return the value of id "1". + So, `AuthorityAdhoc(...)` + """ + # Grabs a dict associated with the file_type. + # For example, EnumFilenames.DOMAIN_ADDITIONAL. + desired_type = self.domain_object.file_data.get(file_type) + if desired_type is None: + self.parse_logs.create_log_item( + file_type, LogCode.ERROR, f"Type {file_type} does not exist" + ) + return None + + # Grab the value given an Id within that file_type dict. + # For example, "igorville.gov". + obj = desired_type.get(desired_id) + if obj is None: self.parse_logs.create_log_item( file_type, LogCode.ERROR, f"Id {desired_id} does not exist" ) + return obj diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 1b0623b35..bc8d1be34 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -216,17 +216,18 @@ class Command(BaseCommand): """ ) - def run_load_transition_domain_script(self, - file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, - sep: str, - reset_table: bool, - debug_on: bool, - prompts_enabled: bool, - debug_max_entries_to_parse: int): - + def run_load_transition_domain_script( + self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int + ): """Runs the load_transition_domain script""" # Create the command string command_script = "load_transition_domain" diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 3fe60da2b..10eb3fee8 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -1,3 +1,10 @@ +""" +A list of helper classes to facilitate handling data from verisign data exports. + +Regarding our dataclasses: +Not intended to be used as models but rather as an alternative to storing as a dictionary. +By keeping it as a dataclass instead of a dictionary, we can maintain data consistency. +""" from dataclasses import dataclass from enum import Enum from typing import Optional @@ -6,7 +13,6 @@ from typing import Optional @dataclass class AgencyAdhoc: """Defines the structure given in the AGENCY_ADHOC file""" - agencyid: Optional[int] = None agencyname: Optional[str] = None active: Optional[str] = None @@ -16,7 +22,6 @@ class AgencyAdhoc: @dataclass class DomainAdditionalData: """Defines the structure given in the DOMAIN_ADDITIONAL file""" - domainname: Optional[str] = None domaintypeid: Optional[int] = None authorityid: Optional[int] = None @@ -29,7 +34,6 @@ class DomainAdditionalData: @dataclass class DomainTypeAdhoc: """Defines the structure given in the DOMAIN_ADHOC file""" - domaintypeid: Optional[int] = None domaintype: Optional[str] = None code: Optional[str] = None @@ -39,7 +43,6 @@ class DomainTypeAdhoc: @dataclass class OrganizationAdhoc: """Defines the structure given in the ORGANIZATION_ADHOC file""" - orgid: Optional[int] = None orgname: Optional[str] = None orgstreet: Optional[str] = None @@ -49,12 +52,29 @@ class OrganizationAdhoc: orgcountrycode: Optional[str] = None +@dataclass +class AuthorityAdhoc: + """Defines the structure given in the AUTHORITY_ADHOC file""" + authorityid: Optional[int] = None + firstname: Optional[str] = None + middlename: Optional[str] = None + lastname: Optional[str] = None + email: Optional[str] = None + phonenumber: Optional[str] = None + agencyid: Optional[int] = None + addlinfo: Optional[str] = None + + + class EnumFilenames(Enum): """Returns a tuple mapping for (filetype, default_file_name). For instance, AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") """ + # We are sourcing data from many different locations, so its better to track this + # as an Enum rather than multiple spread out variables. + # We store the "type" as [0], and we store the "default_filepath" as [1]. AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") DOMAIN_ADDITIONAL = ( "domain_additional", @@ -62,3 +82,4 @@ class EnumFilenames(Enum): ) DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") + AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt") diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index 2010fe563..02879535d 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -12,6 +12,7 @@ from epp_data_containers import ( DomainAdditionalData, DomainTypeAdhoc, OrganizationAdhoc, + AuthorityAdhoc, EnumFilenames, ) @@ -67,15 +68,19 @@ class PatternMap: date = match.group(1) filename_without_date = match.group(2) + # Can the supplied self.regex do a match on the filename? can_infer = filename_without_date == default_file_name if not can_infer: return (self.filename, False) + # If so, note that and return the inferred name full_filename = date + filename_without_date return (full_filename, can_infer) class ExtraTransitionDomain: + """Helper class to aid in storing TransitionDomain data spread across + multiple files.""" filenames = EnumFilenames strip_date_regex = re.compile(r"\d+\.(.+)") @@ -85,16 +90,18 @@ class ExtraTransitionDomain: domain_additional_filename=filenames.DOMAIN_ADDITIONAL[1], domain_adhoc_filename=filenames.DOMAIN_ADHOC[1], organization_adhoc_filename=filenames.ORGANIZATION_ADHOC[1], + authority_adhoc_filename=filenames.AUTHORITY_ADHOC[1], directory="migrationdata", seperator="|", ): self.directory = directory self.seperator = seperator - self.all_files = glob.glob(f"{directory}/*") - # Create a set with filenames as keys for quick lookup - self.all_files_set = {os.path.basename(file) for file in self.all_files} - self.csv_data = { + _all_files = glob.glob(f"{directory}/*") + # Create a set with filenames as keys for quick lookup + self.all_files_set = {os.path.basename(file) for file in _all_files} + + self.file_data = { # (filename, default_url): metadata about the desired file self.filenames.AGENCY_ADHOC: PatternMap( agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc, "agencyid" @@ -117,16 +124,22 @@ class ExtraTransitionDomain: OrganizationAdhoc, "orgid", ), + self.filenames.AUTHORITY_ADHOC: PatternMap( + authority_adhoc_filename, + self.strip_date_regex, + AuthorityAdhoc, + "authorityid", + ), } - def parse_all_files(self, overwrite_existing_data=True): + def parse_all_files(self): """Clears all preexisting data then parses each related CSV file. overwrite_existing_data: bool -> Determines if we should clear - csv_data.data if it already exists + file_data.data if it already exists """ - self.clear_csv_data() - for item in self.csv_data: + self.clear_file_data() + for item in self.file_data: file_type: PatternMap = item.value filename = file_type.filename @@ -141,8 +154,8 @@ class ExtraTransitionDomain: # Log if we can't find the desired file logger.error(f"Could not find file: {filename}") - def clear_csv_data(self): - for item in self.csv_data: + def clear_file_data(self): + for item in self.file_data: file_type: PatternMap = item.value file_type.data = {} diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index d95d8e441..aca80881c 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -48,6 +48,12 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Type of organization", ) + organization_name = models.TextField( + null=True, + blank=True, + help_text="Organization name", + db_index=True, + ) federal_type = models.TextField( max_length=50, null=True, From cb4db4f71aedea0555a7f3b89852c8cebc55e6ac Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:29:56 -0600 Subject: [PATCH 09/88] Script changes --- .../commands/load_extra_transition_domain.py | 55 +++++++---- .../utility/extra_transition_domain.py | 96 +++++++++++++------ ...ansitiondomain_federal_agency_and_more.py} | 9 +- 3 files changed, 109 insertions(+), 51 deletions(-) rename src/registrar/migrations/{0043_transitiondomain_federal_agency_and_more.py => 0044_transitiondomain_federal_agency_and_more.py} (75%) diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index f219081e1..640653697 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -51,14 +51,14 @@ class FileTransitionLog: } class LogItem: - """Used for storing data about logger information. - Intended for use in""" - def __init__(self, file_type, code, message): + """Used for storing data about logger information.""" + def __init__(self, file_type, code, message, domain_name): self.file_type = file_type self.code = code self.message = message + self.domain_name = domain_name - def add_log(self, file_type, code, message): + def add_log(self, file_type, code, message, domain_name): """Adds a log item to self.logs file_type -> Which array to add to, @@ -68,18 +68,18 @@ class FileTransitionLog: message -> Message to display """ - self.logs[file_type] = self.LogItem(file_type, code, message) + self.logs[file_type].append(self.LogItem(file_type, code, message, domain_name)) - def create_log_item(self, file_type, code, message, add_to_list=True): + def create_log_item(self, file_type, code, message, domain_name=None, add_to_list=True): """Creates and returns an LogItem object. add_to_list: bool -> If enabled, add it to the logs array. """ - log = self.LogItem(file_type, code, message) + log = self.LogItem(file_type, code, message, domain_name) if not add_to_list: return log else: - self.logs[file_type] = log + self.logs[file_type].append(log) return log def display_logs(self, file_type): @@ -89,7 +89,8 @@ class FileTransitionLog: for log in self.logs.get(file_type): match log.code: case LogCode.ERROR: - logger.error(log.message) + if log.domain_name is None: + logger.error(log.message) case LogCode.WARNING: logger.warning(log.message) case LogCode.INFO: @@ -110,22 +111,22 @@ class Command(BaseCommand): ) parser.add_argument( "--agency_adhoc_filename", - default=EnumFilenames.AGENCY_ADHOC[1], + default=EnumFilenames.AGENCY_ADHOC.value[1], help="Defines the filename for agency adhocs", ) parser.add_argument( "--domain_additional_filename", - default=EnumFilenames.DOMAIN_ADDITIONAL[1], + default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], help="Defines the filename for additional domain data", ) parser.add_argument( "--domain_adhoc_filename", - default=EnumFilenames.DOMAIN_ADHOC[1], + default=EnumFilenames.DOMAIN_ADHOC.value[1], help="Defines the filename for domain type adhocs", ) parser.add_argument( "--organization_adhoc_filename", - default=EnumFilenames.ORGANIZATION_ADHOC[1], + default=EnumFilenames.ORGANIZATION_ADHOC.value[1], help="Defines the filename for domain type adhocs", ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -143,6 +144,7 @@ class Command(BaseCommand): self.domain_object.parse_all_files() except Exception as err: logger.error(f"Could not load additional data. Error: {err}") + raise err else: all_transition_domains = TransitionDomain.objects.all() if not all_transition_domains.exists(): @@ -190,8 +192,9 @@ class Command(BaseCommand): if info is None: self.parse_logs.create_log_item( EnumFilenames.AGENCY_ADHOC, - LogCode.INFO, - f"Could not add federal_agency on {domain_name}, no data exists." + LogCode.ERROR, + f"Could not add federal_agency on {domain_name}, no data exists.", + domain_name ) return transition_domain @@ -205,6 +208,7 @@ class Command(BaseCommand): EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive agency {info.agencyname} on {domain_name}", + domain_name ) return transition_domain @@ -213,6 +217,7 @@ class Command(BaseCommand): EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add non-federal agency {info.agencyname} on {domain_name}", + domain_name ) return transition_domain @@ -242,8 +247,9 @@ class Command(BaseCommand): if info is None: self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, - LogCode.INFO, + LogCode.ERROR, f"Could not add domain_type on {domain_name}, no data exists.", + domain_name ) return transition_domain @@ -264,6 +270,7 @@ class Command(BaseCommand): EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", + domain_name ) return transition_domain @@ -317,8 +324,9 @@ class Command(BaseCommand): if org_info is None: self.parse_logs.create_log_item( EnumFilenames.ORGANIZATION_ADHOC, - LogCode.INFO, + LogCode.ERROR, f"Could not add organization_name on {domain_name}, no data exists.", + domain_name ) return transition_domain @@ -351,32 +359,42 @@ class Command(BaseCommand): file_type, LogCode.DEBUG, f"Added {file_type} as '{var_name}' on {domain_name}", + domain_name ) else: self.parse_logs.create_log_item( file_type, LogCode.INFO, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", + domain_name ) # Property getters, i.e. orgid or domaintypeid def get_org_info(self, domain_name) -> OrganizationAdhoc: domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None org_id = domain_info.orgid return self.get_organization_adhoc(org_id) def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None type_id = domain_info.domaintypeid return self.get_domain_adhoc(type_id) def get_agency_info(self, domain_name) -> AgencyAdhoc: domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None type_id = domain_info.orgid return self.get_domain_adhoc(type_id) def get_authority_info(self, domain_name): domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None type_id = domain_info.authorityid return self.get_authority_adhoc(type_id) @@ -441,10 +459,9 @@ class Command(BaseCommand): # Grab the value given an Id within that file_type dict. # For example, "igorville.gov". - obj = desired_type.get(desired_id) + obj = desired_type.data.get(desired_id) if obj is None: self.parse_logs.create_log_item( file_type, LogCode.ERROR, f"Id {desired_id} does not exist" ) - return obj diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index 02879535d..c35b9aa4c 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -7,7 +7,7 @@ import logging import os from typing import List -from epp_data_containers import ( +from .epp_data_containers import ( AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, @@ -45,23 +45,26 @@ class PatternMap: regex: re.Pattern, data_type: type, id_field: str, - data: dict = {}, ): self.regex = regex self.data_type = data_type self.id_field = id_field - self.data = data + self.data = {} + self.filename = filename + self.could_infer = False + def try_infer_filename(self, current_file_name, default_file_name): + """Tries to match a given filename to a regex, + then uses that match to generate the filename.""" # returns (filename, inferred_successfully) - _infer = self._infer_filename(self.regex, filename) - self.filename = _infer[0] - self.could_infer = _infer[1] + return self._infer_filename(self.regex, current_file_name, default_file_name) - def _infer_filename(self, regex: re.Pattern, default_file_name): + def _infer_filename(self, regex: re.Pattern, matched_file_name, default_file_name): if not isinstance(regex, re.Pattern): return (self.filename, False) - - match = regex.match(self.filename) + + match = regex.match(matched_file_name) + if not match: return (self.filename, False) @@ -74,7 +77,7 @@ class PatternMap: return (self.filename, False) # If so, note that and return the inferred name - full_filename = date + filename_without_date + full_filename = date + "." + filename_without_date return (full_filename, can_infer) @@ -82,25 +85,28 @@ class ExtraTransitionDomain: """Helper class to aid in storing TransitionDomain data spread across multiple files.""" filenames = EnumFilenames - strip_date_regex = re.compile(r"\d+\.(.+)") + #strip_date_regex = re.compile(r"\d+\.(.+)") + strip_date_regex = re.compile(r"(?:.*\/)?(\d+)\.(.+)") def __init__( self, - agency_adhoc_filename=filenames.AGENCY_ADHOC[1], - domain_additional_filename=filenames.DOMAIN_ADDITIONAL[1], - domain_adhoc_filename=filenames.DOMAIN_ADHOC[1], - organization_adhoc_filename=filenames.ORGANIZATION_ADHOC[1], - authority_adhoc_filename=filenames.AUTHORITY_ADHOC[1], + agency_adhoc_filename=filenames.AGENCY_ADHOC.value[1], + domain_additional_filename=filenames.DOMAIN_ADDITIONAL.value[1], + domain_adhoc_filename=filenames.DOMAIN_ADHOC.value[1], + organization_adhoc_filename=filenames.ORGANIZATION_ADHOC.value[1], + authority_adhoc_filename=filenames.AUTHORITY_ADHOC.value[1], directory="migrationdata", seperator="|", ): + # Add a slash if the last character isn't one + if directory and directory[-1] != "/": + directory += "/" self.directory = directory self.seperator = seperator - _all_files = glob.glob(f"{directory}/*") + self.all_files = glob.glob(f"{directory}*") # Create a set with filenames as keys for quick lookup - self.all_files_set = {os.path.basename(file) for file in _all_files} - + self.all_files_set = {os.path.basename(file) for file in self.all_files} self.file_data = { # (filename, default_url): metadata about the desired file self.filenames.AGENCY_ADHOC: PatternMap( @@ -132,34 +138,62 @@ class ExtraTransitionDomain: ), } - def parse_all_files(self): + def parse_all_files(self, infer_filenames=True): """Clears all preexisting data then parses each related CSV file. overwrite_existing_data: bool -> Determines if we should clear file_data.data if it already exists """ self.clear_file_data() - for item in self.file_data: - file_type: PatternMap = item.value - filename = file_type.filename + for name, value in self.file_data.items(): + filename = f"{value.filename}" if filename in self.all_files_set: - file_type.data = self._read_csv_file( - self.all_files_set[filename], + _file = f"{self.directory}{value.filename}" + value.data = self._read_csv_file( + _file, self.seperator, - file_type.data_type, - file_type.id_field, + value.data_type, + value.id_field, ) else: + if not infer_filenames: + logger.error(f"Could not find file: {filename}") + continue + + logger.warning( + "Attempting to infer filename" + f" for file: {filename}." + ) + for filename in self.all_files: + default_name = name.value[1] + match = value.try_infer_filename(filename, default_name) + filename = match[0] + can_infer = match[1] + if can_infer: + break + + if filename in self.all_files_set: + logger.info(f"Infer success. Found file {filename}") + _file = f"{self.directory}{filename}" + value.data = self._read_csv_file( + _file, + self.seperator, + value.data_type, + value.id_field, + ) + continue # Log if we can't find the desired file logger.error(f"Could not find file: {filename}") def clear_file_data(self): - for item in self.file_data: - file_type: PatternMap = item.value + for item in self.file_data.values(): + file_type: PatternMap = item file_type.data = {} def _read_csv_file(self, file, seperator, dataclass_type, id_field): - with open(file, "r", encoding="utf-8") as requested_file: + with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) - return {row[id_field]: dataclass_type(**row) for row in reader} + dict_data = {row[id_field]: dataclass_type(**row) for row in reader} + logger.debug(f"it is finally here {dict_data}") + return dict_data diff --git a/src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py b/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py similarity index 75% rename from src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py rename to src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py index 361b91789..4de69695b 100644 --- a/src/registrar/migrations/0043_transitiondomain_federal_agency_and_more.py +++ b/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py @@ -5,7 +5,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("registrar", "0042_create_groups_v03"), + ("registrar", "0043_domain_expiration_date"), ] operations = [ @@ -31,4 +31,11 @@ class Migration(migrations.Migration): blank=True, help_text="Type of organization", max_length=255, null=True ), ), + migrations.AddField( + model_name="transitiondomain", + name="organization_name", + field=models.TextField( + blank=True, db_index=True, help_text="Organization name", null=True + ), + ), ] From c9537ed7f5c63689513c8fa65ab2424850d9f93b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:59:13 -0600 Subject: [PATCH 10/88] Finish parsing for most fields minus expiration --- .../commands/load_extra_transition_domain.py | 28 +++++++++---------- .../commands/utility/epp_data_containers.py | 6 ++-- .../utility/extra_transition_domain.py | 19 +++++++++++-- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py index 640653697..a3fdca7e3 100644 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ b/src/registrar/management/commands/load_extra_transition_domain.py @@ -89,8 +89,7 @@ class FileTransitionLog: for log in self.logs.get(file_type): match log.code: case LogCode.ERROR: - if log.domain_name is None: - logger.error(log.message) + logger.error(log.message) case LogCode.WARNING: logger.warning(log.message) case LogCode.INFO: @@ -149,11 +148,9 @@ class Command(BaseCommand): all_transition_domains = TransitionDomain.objects.all() if not all_transition_domains.exists(): raise Exception("No TransitionDomain objects exist.") - for transition_domain in all_transition_domains: - domain_name = transition_domain.domain_name + domain_name = transition_domain.domain_name.upper() updated_transition_domain = transition_domain - # STEP 1: Parse organization data updated_transition_domain = self.parse_org_data( domain_name, transition_domain @@ -166,7 +163,7 @@ class Command(BaseCommand): ) self.parse_logs.display_logs(EnumFilenames.DOMAIN_ADHOC) - # STEP 3: Parse agency data - TODO + # STEP 3: Parse agency data updated_transition_domain = self.parse_agency_data( domain_name, transition_domain ) @@ -257,8 +254,9 @@ class Command(BaseCommand): # For all other records, it is stored as so: Interstate # We can infer if it is federal or not based on this fact. domain_type = info.domaintype.split("-") - if domain_type.count != 1 or domain_type.count != 2: - raise ValueError("Found invalid data in DOMAIN_ADHOC") + domain_type_length = len(domain_type) + if domain_type_length < 1 or domain_type_length > 2: + raise ValueError("Found invalid data on DOMAIN_ADHOC") # Then, just grab the organization type. new_organization_type = domain_type[0].strip() @@ -276,9 +274,9 @@ class Command(BaseCommand): # Are we updating data that already exists, # or are we adding new data in its place? - federal_agency_exists = ( + organization_type_exists = ( transition_domain.organization_type is not None - and transition_domain.federal_agency.strip() != "" + and transition_domain.organization_type.strip() != "" ) federal_type_exists = ( transition_domain.federal_type is not None @@ -287,7 +285,7 @@ class Command(BaseCommand): # If we get two records, then we know it is federal. # needs to be lowercase for federal type - is_federal = domain_type.count() == 2 + is_federal = domain_type_length == 2 if is_federal: new_federal_type = domain_type[1].strip() transition_domain.organization_type = new_organization_type @@ -300,10 +298,10 @@ class Command(BaseCommand): # or modified it. self._add_or_change_message( EnumFilenames.DOMAIN_ADHOC, - "federal_agency", - transition_domain.federal_agency, + "organization_type", + transition_domain.organization_type, domain_name, - federal_agency_exists, + organization_type_exists, ) self._add_or_change_message( @@ -358,7 +356,7 @@ class Command(BaseCommand): self.parse_logs.create_log_item( file_type, LogCode.DEBUG, - f"Added {file_type} as '{var_name}' on {domain_name}", + f"Added {var_name} as '{changed_value}' on {domain_name}", domain_name ) else: diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 10eb3fee8..1e5506029 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -7,7 +7,7 @@ By keeping it as a dataclass instead of a dictionary, we can maintain data consi """ from dataclasses import dataclass from enum import Enum -from typing import Optional +from typing import List, Optional @dataclass @@ -26,7 +26,7 @@ class DomainAdditionalData: domaintypeid: Optional[int] = None authorityid: Optional[int] = None orgid: Optional[int] = None - securitycontact_email: Optional[str] = None + securitycontactemail: Optional[str] = None dnsseckeymonitor: Optional[str] = None domainpurpose: Optional[str] = None @@ -62,7 +62,7 @@ class AuthorityAdhoc: email: Optional[str] = None phonenumber: Optional[str] = None agencyid: Optional[int] = None - addlinfo: Optional[str] = None + addlinfo: Optional[List[str]] = None diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index c35b9aa4c..aea782697 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -146,8 +146,8 @@ class ExtraTransitionDomain: """ self.clear_file_data() for name, value in self.file_data.items(): - filename = f"{value.filename}" + filename = f"{value.filename}" if filename in self.all_files_set: _file = f"{self.directory}{value.filename}" value.data = self._read_csv_file( @@ -194,6 +194,19 @@ class ExtraTransitionDomain: def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) - dict_data = {row[id_field]: dataclass_type(**row) for row in reader} - logger.debug(f"it is finally here {dict_data}") + """ + for row in reader: + print({key: type(key) for key in row.keys()}) # print out the keys and their types + test = {row[id_field]: dataclass_type(**row)} + """ + dict_data = {} + for row in reader: + if None in row: + print("Skipping row with None key") + #for key, value in row.items(): + #print(f"key: {key} value: {value}") + continue + row_id = row[id_field] + dict_data[row_id] = dataclass_type(**row) + #dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data From 6c75ba44c5287497a39281100d82bb8b9d567e87 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 31 Oct 2023 12:08:34 -0600 Subject: [PATCH 11/88] Move logic into its own class --- .../commands/utility/epp_data_containers.py | 1 - .../utility/extra_transition_domain.py | 446 +++++++++++++++++- .../utility/transition_domain_arguments.py | 26 + 3 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 src/registrar/management/commands/utility/transition_domain_arguments.py diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 1e5506029..8fed465d9 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -65,7 +65,6 @@ class AuthorityAdhoc: addlinfo: Optional[List[str]] = None - class EnumFilenames(Enum): """Returns a tuple mapping for (filetype, default_file_name). diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index aea782697..d6b2d360b 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -1,13 +1,17 @@ """""" import csv from dataclasses import dataclass +from enum import Enum import glob import re import logging import os from typing import List -from .epp_data_containers import ( + +from registrar.models.transition_domain import TransitionDomain +from transition_domain_arguments import TransitionDomainArguments +from epp_data_containers import ( AgencyAdhoc, DomainAdditionalData, DomainTypeAdhoc, @@ -18,6 +22,444 @@ from .epp_data_containers import ( logger = logging.getLogger(__name__) +class LogCode(Enum): + """Stores the desired log severity""" + ERROR = 1 + WARNING = 2 + INFO = 3 + DEBUG = 4 + + +class FileTransitionLog: + """Container for storing event logs. Used to lessen + the complexity of storing multiple logs across multiple + variables. + + self.logs: dict -> { + EnumFilenames.DOMAIN_ADHOC: List[LogItem], + EnumFilenames.AGENCY_ADHOC: List[LogItem], + EnumFilenames.ORGANIZATION_ADHOC: List[LogItem], + EnumFilenames.DOMAIN_ADDITIONAL: List[LogItem], + } + """ + def __init__(self): + self.logs = { + EnumFilenames.DOMAIN_ADHOC: [], + EnumFilenames.AGENCY_ADHOC: [], + EnumFilenames.ORGANIZATION_ADHOC: [], + EnumFilenames.DOMAIN_ADDITIONAL: [], + } + + class LogItem: + """Used for storing data about logger information.""" + def __init__(self, file_type, code, message, domain_name): + self.file_type = file_type + self.code = code + self.message = message + self.domain_name = domain_name + + def add_log(self, file_type, code, message, domain_name): + """Adds a log item to self.logs + + file_type -> Which array to add to, + ex. EnumFilenames.DOMAIN_ADHOC + + code -> Log severity or other metadata, ex. LogCode.ERROR + + message -> Message to display + """ + self.logs[file_type].append(self.LogItem(file_type, code, message, domain_name)) + + def create_log_item(self, file_type, code, message, domain_name=None, add_to_list=True): + """Creates and returns an LogItem object. + + add_to_list: bool -> If enabled, add it to the logs array. + """ + log = self.LogItem(file_type, code, message, domain_name) + if not add_to_list: + return log + else: + self.logs[file_type].append(log) + return log + + def display_logs(self, file_type): + """Displays all logs in the given file_type in EnumFilenames. + Will log with the correct severity depending on code. + """ + for log in self.logs.get(file_type): + match log.code: + case LogCode.ERROR: + logger.error(log.message) + case LogCode.WARNING: + logger.warning(log.message) + case LogCode.INFO: + logger.info(log.message) + case LogCode.DEBUG: + logger.debug(log.message) + + +class LoadExtraTransitionDomain: + """Grabs additional data for TransitionDomains.""" + + def __init__(self, options: TransitionDomainArguments): + # Stores event logs and organizes them + self.parse_logs = FileTransitionLog() + + # Reads and parses migration files + self.domain_object = ExtraTransitionDomain( + agency_adhoc_filename=options.agency_adhoc_filename, + domain_additional_filename=options.domain_additional_filename, + domain_adhoc_filename=options.domain_adhoc_filename, + organization_adhoc_filename=options.organization_adhoc_filename, + directory=options.directory, + seperator=options.seperator, + ) + self.domain_object.parse_all_files() + + # Given the data we just parsed, update each + # transition domain object with that data. + self.update_transition_domain_models() + + + def update_transition_domain_models(self): + """Updates TransitionDomain objects based off the file content + given in self.domain_object""" + all_transition_domains = TransitionDomain.objects.all() + if not all_transition_domains.exists(): + raise Exception("No TransitionDomain objects exist.") + + for transition_domain in all_transition_domains: + domain_name = transition_domain.domain_name.upper() + updated_transition_domain = transition_domain + # STEP 1: Parse organization data + updated_transition_domain = self.parse_org_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.ORGANIZATION_ADHOC) + + # STEP 2: Parse domain type data + updated_transition_domain = self.parse_domain_type_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.DOMAIN_ADHOC) + + # STEP 3: Parse agency data + updated_transition_domain = self.parse_agency_data( + domain_name, transition_domain + ) + self.parse_logs.display_logs(EnumFilenames.AGENCY_ADHOC) + + # STEP 4: Parse expiration data - TODO + updated_transition_domain = self.parse_expiration_data( + domain_name, transition_domain + ) + # self.parse_logs(EnumFilenames.EXPIRATION_DATA) + + updated_transition_domain.save() + + # TODO - Implement once Niki gets her ticket in + def parse_expiration_data(self, domain_name, transition_domain): + """Grabs expiration_date from the parsed files and associates it + with a transition_domain object, then returns that object.""" + return transition_domain + + def parse_agency_data(self, domain_name, transition_domain) -> TransitionDomain: + """Grabs federal_agency from the parsed files and associates it + with a transition_domain object, then returns that object.""" + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + info = self.get_agency_info(domain_name) + if info is None: + self.parse_logs.create_log_item( + EnumFilenames.AGENCY_ADHOC, + LogCode.ERROR, + f"Could not add federal_agency on {domain_name}, no data exists.", + domain_name + ) + return transition_domain + + agency_exists = ( + transition_domain.federal_agency is not None + and transition_domain.federal_agency.strip() != "" + ) + + if not info.active.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add inactive agency {info.agencyname} on {domain_name}", + domain_name + ) + return transition_domain + + if not info.isfederal.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add non-federal agency {info.agencyname} on {domain_name}", + domain_name + ) + return transition_domain + + transition_domain.federal_agency = info.agencyname + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.AGENCY_ADHOC, + "federal_agency", + transition_domain.federal_agency, + domain_name, + agency_exists + ) + + return transition_domain + + def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: + """Grabs organization_type and federal_type from the parsed files + and associates it with a transition_domain object, then returns that object.""" + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + info = self.get_domain_type_info(domain_name) + if info is None: + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add domain_type on {domain_name}, no data exists.", + domain_name + ) + return transition_domain + + # This data is stored as follows: FEDERAL - Judicial + # For all other records, it is stored as so: Interstate + # We can infer if it is federal or not based on this fact. + domain_type = info.domaintype.split("-") + domain_type_length = len(domain_type) + if domain_type_length < 1 or domain_type_length > 2: + raise ValueError("Found invalid data on DOMAIN_ADHOC") + + # Then, just grab the organization type. + new_organization_type = domain_type[0].strip() + + # Check if this domain_type is active or not. + # If not, we don't want to add this. + if not info.active.lower() == "y": + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ADHOC, + LogCode.ERROR, + f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", + domain_name + ) + return transition_domain + + # Are we updating data that already exists, + # or are we adding new data in its place? + organization_type_exists = ( + transition_domain.organization_type is not None + and transition_domain.organization_type.strip() != "" + ) + federal_type_exists = ( + transition_domain.federal_type is not None + and transition_domain.federal_type.strip() != "" + ) + + # If we get two records, then we know it is federal. + # needs to be lowercase for federal type + is_federal = domain_type_length == 2 + if is_federal: + new_federal_type = domain_type[1].strip() + transition_domain.organization_type = new_organization_type + transition_domain.federal_type = new_federal_type + else: + transition_domain.organization_type = new_organization_type + transition_domain.federal_type = None + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.DOMAIN_ADHOC, + "organization_type", + transition_domain.organization_type, + domain_name, + organization_type_exists, + ) + + self._add_or_change_message( + EnumFilenames.DOMAIN_ADHOC, + "federal_type", + transition_domain.federal_type, + domain_name, + federal_type_exists, + ) + + return transition_domain + + def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: + """Grabs organization_name from the parsed files and associates it + with a transition_domain object, then returns that object.""" + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + org_info = self.get_org_info(domain_name) + if org_info is None: + self.parse_logs.create_log_item( + EnumFilenames.ORGANIZATION_ADHOC, + LogCode.ERROR, + f"Could not add organization_name on {domain_name}, no data exists.", + domain_name + ) + return transition_domain + + desired_property_exists = ( + transition_domain.organization_name is not None + and transition_domain.organization_name.strip() != "" + ) + + transition_domain.organization_name = org_info.orgname + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.ORGANIZATION_ADHOC, + "organization_name", + transition_domain.organization_name, + domain_name, + desired_property_exists, + ) + + return transition_domain + + def _add_or_change_message( + self, file_type, var_name, changed_value, domain_name, is_update=False + ): + """Creates a log instance when a property + is successfully changed on a given TransitionDomain.""" + if not is_update: + self.parse_logs.create_log_item( + file_type, + LogCode.DEBUG, + f"Added {var_name} as '{changed_value}' on {domain_name}", + domain_name + ) + else: + self.parse_logs.create_log_item( + file_type, + LogCode.INFO, + f"Updated existing {var_name} to '{changed_value}' on {domain_name}", + domain_name + ) + + # Property getters, i.e. orgid or domaintypeid + def get_org_info(self, domain_name) -> OrganizationAdhoc: + """Maps an id given in get_domain_data to a organization_adhoc + record which has its corresponding definition""" + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + org_id = domain_info.orgid + return self.get_organization_adhoc(org_id) + + def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: + """Maps an id given in get_domain_data to a domain_type_adhoc + record which has its corresponding definition""" + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + type_id = domain_info.domaintypeid + return self.get_domain_adhoc(type_id) + + def get_agency_info(self, domain_name) -> AgencyAdhoc: + """Maps an id given in get_domain_data to a agency_adhoc + record which has its corresponding definition""" + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + type_id = domain_info.orgid + return self.get_domain_adhoc(type_id) + + def get_authority_info(self, domain_name): + """Maps an id given in get_domain_data to a authority_adhoc + record which has its corresponding definition""" + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + type_id = domain_info.authorityid + return self.get_authority_adhoc(type_id) + + # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc + def get_domain_data(self, desired_id) -> DomainAdditionalData: + """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) + + def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: + """Grabs a corresponding row within the ORGANIZATION_ADHOC file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) + + def get_domain_adhoc(self, desired_id) -> DomainTypeAdhoc: + """Grabs a corresponding row within the DOMAIN_ADHOC file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.DOMAIN_ADHOC, desired_id) + + def get_agency_adhoc(self, desired_id) -> AgencyAdhoc: + """Grabs a corresponding row within the AGENCY_ADHOC file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) + + def get_authority_adhoc(self, desired_id) -> AuthorityAdhoc: + """Grabs a corresponding row within the AUTHORITY_ADHOC file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) + + def get_object_by_id(self, file_type: EnumFilenames, desired_id): + """Returns a field in a dictionary based off the type and id. + + vars: + file_type: (constant) EnumFilenames -> Which data file to target. + An example would be `EnumFilenames.DOMAIN_ADHOC`. + + desired_id: str -> Which id you want to search on. + An example would be `"12"` or `"igorville.gov"` + + Explanation: + Each data file has an associated type (file_type) for tracking purposes. + + Each file_type is a dictionary which + contains a dictionary of row[id_field]: object. + + In practice, this would look like: + + EnumFilenames.AUTHORITY_ADHOC: { + "1": AuthorityAdhoc(...), + "2": AuthorityAdhoc(...), + ... + } + + desired_id will then specify which id to grab. If we wanted "1", + then this function will return the value of id "1". + So, `AuthorityAdhoc(...)` + """ + # Grabs a dict associated with the file_type. + # For example, EnumFilenames.DOMAIN_ADDITIONAL. + desired_type = self.domain_object.file_data.get(file_type) + if desired_type is None: + self.parse_logs.create_log_item( + file_type, LogCode.ERROR, f"Type {file_type} does not exist" + ) + return None + + # Grab the value given an Id within that file_type dict. + # For example, "igorville.gov". + obj = desired_type.data.get(desired_id) + if obj is None: + self.parse_logs.create_log_item( + file_type, LogCode.ERROR, f"Id {desired_id} does not exist" + ) + return obj + @dataclass class PatternMap: @@ -209,4 +651,4 @@ class ExtraTransitionDomain: row_id = row[id_field] dict_data[row_id] = dataclass_type(**row) #dict_data = {row[id_field]: dataclass_type(**row) for row in reader} - return dict_data + return dict_data \ No newline at end of file diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py new file mode 100644 index 000000000..44beab5b2 --- /dev/null +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + + +@dataclass +class TransitionDomainArguments: + """Stores arguments for load_transition_domain""" + # Settings # + directory: str + seperator: str + limit_parse: int + + # Filenames # + ## Adhocs ## + agency_adhoc_filename: str + domain_adhoc_filename: str + organization_adhoc_filename: str + + ## Data files ## + domain_additional_filename: str + domain_contacts_filename: str + domain_statuses_filename: str + + # Flags # + debug: bool + reset_table: bool + load_extra: bool \ No newline at end of file From 13172870fbea2ac4d63e1b0ee6875a2350dd2f95 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:09:19 -0600 Subject: [PATCH 12/88] Prepare for parsing expiration date --- .../commands/load_transition_domain.py | 45 ++++- .../management/commands/utility/__init__.py | 0 .../commands/utility/epp_data_containers.py | 9 + .../utility/extra_transition_domain.py | 155 ++++++++++++------ .../utility/transition_domain_arguments.py | 51 +++--- 5 files changed, 189 insertions(+), 71 deletions(-) create mode 100644 src/registrar/management/commands/utility/__init__.py diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 34fdfb50c..3a08d70dd 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -6,6 +6,7 @@ import argparse from collections import defaultdict from django.core.management import BaseCommand +from registrar.management.commands.utility.epp_data_containers import EnumFilenames from registrar.models import TransitionDomain @@ -14,6 +15,9 @@ from registrar.management.commands.utility.terminal_helper import ( TerminalHelper, ) +from .utility.transition_domain_arguments import TransitionDomainArguments +from .utility.extra_transition_domain import LoadExtraTransitionDomain + logger = logging.getLogger(__name__) @@ -61,6 +65,31 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) + # TODO - Narrow this down + parser.add_argument( + "--directory", default="migrationdata", help="Desired directory" + ) + parser.add_argument( + "--agency_adhoc_filename", + default=EnumFilenames.AGENCY_ADHOC.value[1], + help="Defines the filename for agency adhocs", + ) + parser.add_argument( + "--domain_additional_filename", + default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], + help="Defines the filename for additional domain data", + ) + parser.add_argument( + "--domain_adhoc_filename", + default=EnumFilenames.DOMAIN_ADHOC.value[1], + help="Defines the filename for domain type adhocs", + ) + parser.add_argument( + "--organization_adhoc_filename", + default=EnumFilenames.ORGANIZATION_ADHOC.value[1], + help="Defines the filename for domain type adhocs", + ) + def print_debug_mode_statements( self, debug_on: bool, debug_max_entries_to_parse: int ): @@ -255,7 +284,6 @@ class Command(BaseCommand): ): """Parse the data files and create TransitionDomains.""" sep = options.get("sep") - load_extra_data = options.get("loadExtraData") # If --resetTable was used, prompt user to confirm # deletion of table data @@ -286,7 +314,6 @@ class Command(BaseCommand): # STEP 3: # Parse the domain_contacts file and create TransitionDomain objects, # using the dictionaries from steps 1 & 2 to lookup needed information. - to_create = [] # keep track of statuses that don't match our available @@ -472,3 +499,17 @@ class Command(BaseCommand): duplicate_domain_user_combos, duplicate_domains, users_without_email ) self.print_summary_status_findings(domains_without_status, outlier_statuses) + + # Prompt the user if they want to load additional data on the domains + # TODO - add this logic into the core of this file + arguments = TransitionDomainArguments(**options) + + do_parse_extra = TerminalHelper.prompt_for_execution( + True, + "./manage.py test", + "Running load_extra_transition_domains script", + ) + if do_parse_extra: + extra = LoadExtraTransitionDomain(arguments) + extra_logs = extra.parse_logs.logs + diff --git a/src/registrar/management/commands/utility/__init__.py b/src/registrar/management/commands/utility/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 8fed465d9..42dbdebd5 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -6,6 +6,7 @@ Not intended to be used as models but rather as an alternative to storing as a d By keeping it as a dataclass instead of a dictionary, we can maintain data consistency. """ from dataclasses import dataclass +from datetime import date from enum import Enum from typing import List, Optional @@ -64,6 +65,13 @@ class AuthorityAdhoc: agencyid: Optional[int] = None addlinfo: Optional[List[str]] = None +@dataclass +class DomainEscrow: + """Defines the structure given in the DOMAIN_ESCROW file""" + domainname: Optional[str] = None + creationdate: Optional[date] = None + expirationdate: Optional[date] = None + class EnumFilenames(Enum): """Returns a tuple mapping for (filetype, default_file_name). @@ -79,6 +87,7 @@ class EnumFilenames(Enum): "domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt", ) + DOMAIN_ESCROW = ("domain_escrow", "escrow_domains.daily.dotgov.GOV.txt") DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt") diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain.py index d6b2d360b..3489e55c1 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain.py @@ -1,25 +1,29 @@ """""" import csv from dataclasses import dataclass +from datetime import datetime from enum import Enum import glob import re import logging import os -from typing import List +from typing import List, Tuple from registrar.models.transition_domain import TransitionDomain -from transition_domain_arguments import TransitionDomainArguments -from epp_data_containers import ( + +from .epp_data_containers import ( AgencyAdhoc, DomainAdditionalData, + DomainEscrow, DomainTypeAdhoc, OrganizationAdhoc, AuthorityAdhoc, EnumFilenames, ) +from .transition_domain_arguments import TransitionDomainArguments + logger = logging.getLogger(__name__) class LogCode(Enum): @@ -105,15 +109,9 @@ class LoadExtraTransitionDomain: # Stores event logs and organizes them self.parse_logs = FileTransitionLog() + arguments = options.args_extra_transition_domain() # Reads and parses migration files - self.domain_object = ExtraTransitionDomain( - agency_adhoc_filename=options.agency_adhoc_filename, - domain_additional_filename=options.domain_additional_filename, - domain_adhoc_filename=options.domain_adhoc_filename, - organization_adhoc_filename=options.organization_adhoc_filename, - directory=options.directory, - seperator=options.seperator, - ) + self.domain_object = ExtraTransitionDomain(**arguments) self.domain_object.parse_all_files() # Given the data we just parsed, update each @@ -131,6 +129,7 @@ class LoadExtraTransitionDomain: for transition_domain in all_transition_domains: domain_name = transition_domain.domain_name.upper() updated_transition_domain = transition_domain + # STEP 1: Parse organization data updated_transition_domain = self.parse_org_data( domain_name, transition_domain @@ -526,59 +525,91 @@ class PatternMap: class ExtraTransitionDomain: """Helper class to aid in storing TransitionDomain data spread across multiple files.""" - filenames = EnumFilenames - #strip_date_regex = re.compile(r"\d+\.(.+)") strip_date_regex = re.compile(r"(?:.*\/)?(\d+)\.(.+)") def __init__( self, - agency_adhoc_filename=filenames.AGENCY_ADHOC.value[1], - domain_additional_filename=filenames.DOMAIN_ADDITIONAL.value[1], - domain_adhoc_filename=filenames.DOMAIN_ADHOC.value[1], - organization_adhoc_filename=filenames.ORGANIZATION_ADHOC.value[1], - authority_adhoc_filename=filenames.AUTHORITY_ADHOC.value[1], + agency_adhoc_filename=EnumFilenames.AGENCY_ADHOC.value[1], + domain_additional_filename=EnumFilenames.DOMAIN_ADDITIONAL.value[1], + domain_escrow_filename=EnumFilenames.DOMAIN_ESCROW.value[1], + domain_adhoc_filename=EnumFilenames.DOMAIN_ADHOC.value[1], + organization_adhoc_filename=EnumFilenames.ORGANIZATION_ADHOC.value[1], + authority_adhoc_filename=EnumFilenames.AUTHORITY_ADHOC.value[1], directory="migrationdata", - seperator="|", + sep="|", ): # Add a slash if the last character isn't one if directory and directory[-1] != "/": directory += "/" self.directory = directory - self.seperator = seperator + self.seperator = sep self.all_files = glob.glob(f"{directory}*") # Create a set with filenames as keys for quick lookup self.all_files_set = {os.path.basename(file) for file in self.all_files} - self.file_data = { - # (filename, default_url): metadata about the desired file - self.filenames.AGENCY_ADHOC: PatternMap( - agency_adhoc_filename, self.strip_date_regex, AgencyAdhoc, "agencyid" - ), - self.filenames.DOMAIN_ADDITIONAL: PatternMap( - domain_additional_filename, - self.strip_date_regex, - DomainAdditionalData, - "domainname", - ), - self.filenames.DOMAIN_ADHOC: PatternMap( - domain_adhoc_filename, - self.strip_date_regex, - DomainTypeAdhoc, - "domaintypeid", - ), - self.filenames.ORGANIZATION_ADHOC: PatternMap( - organization_adhoc_filename, - self.strip_date_regex, - OrganizationAdhoc, - "orgid", - ), - self.filenames.AUTHORITY_ADHOC: PatternMap( + + # Used for a container of values at each filename. + # Instead of tracking each in a seperate variable, we can declare + # metadata about each file and associate it with an enum. + # That way if we want the data located at the agency_adhoc file, + # we can just call EnumFilenames.AGENCY_ADHOC. + pattern_map_params = [ + (EnumFilenames.AGENCY_ADHOC, agency_adhoc_filename, AgencyAdhoc, "agencyid"), + (EnumFilenames.DOMAIN_ADDITIONAL, domain_additional_filename, DomainAdditionalData, "domainname"), + (EnumFilenames.DOMAIN_ESCROW, domain_escrow_filename, DomainEscrow, "domainname"), + (EnumFilenames.DOMAIN_ADHOC, domain_adhoc_filename, DomainTypeAdhoc, "domaintypeid"), + (EnumFilenames.ORGANIZATION_ADHOC, organization_adhoc_filename, OrganizationAdhoc, "orgid"), + (EnumFilenames.AUTHORITY_ADHOC, authority_adhoc_filename, AuthorityAdhoc, "authorityid"), + ] + self.file_data = self.populate_file_data(pattern_map_params) + + def populate_file_data( + self, + pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] + ): + """Populates the self.file_data field given a set + of tuple params. + + pattern_map_params must adhere to this format: + [ + (field_type, filename, data_type, id_field), + ] + + vars: + file_type (EnumFilenames) -> The name of the dictionary. + Defined as a value on EnumFilenames, such as + EnumFilenames.AGENCY_ADHOC + + filename (str) -> The filepath of the given + "file_type", such as migrationdata/test123.txt + + data_type (type) -> The type of data to be read + at the location of the filename. For instance, + each row of test123.txt may return data of type AgencyAdhoc + + id_field (str) -> Given the "data_type" of each row, + this specifies what the "id" of that row is. + For example, "agencyid". This is used so we can + store each record in a dictionary rather than + a list of values. + + return example: + EnumFilenames.AUTHORITY_ADHOC: PatternMap( authority_adhoc_filename, self.strip_date_regex, AuthorityAdhoc, "authorityid", ), - } + """ + file_data = {} + for file_type, filename, data_type, id_field in pattern_map_params: + file_data[file_type] = PatternMap( + filename, + self.strip_date_regex, + data_type, + id_field, + ) + return file_data def parse_all_files(self, infer_filenames=True): """Clears all preexisting data then parses each related CSV file. @@ -588,15 +619,16 @@ class ExtraTransitionDomain: """ self.clear_file_data() for name, value in self.file_data.items(): - + is_domain_escrow = name == EnumFilenames.DOMAIN_ESCROW filename = f"{value.filename}" if filename in self.all_files_set: _file = f"{self.directory}{value.filename}" - value.data = self._read_csv_file( + value.data = self.parse_csv_file( _file, self.seperator, value.data_type, value.id_field, + is_domain_escrow, ) else: if not infer_filenames: @@ -618,11 +650,12 @@ class ExtraTransitionDomain: if filename in self.all_files_set: logger.info(f"Infer success. Found file {filename}") _file = f"{self.directory}{filename}" - value.data = self._read_csv_file( + value.data = self.parse_csv_file( _file, self.seperator, value.data_type, value.id_field, + is_domain_escrow, ) continue # Log if we can't find the desired file @@ -633,6 +666,32 @@ class ExtraTransitionDomain: file_type: PatternMap = item file_type.data = {} + def parse_csv_file(self, file, seperator, dataclass_type, id_field, is_domain_escrow=False): + # Domain escrow is an edge case + if is_domain_escrow: + return self._read_domain_escrow(file, seperator) + else: + return self._read_csv_file(file, seperator, dataclass_type, id_field) + + # Domain escrow is an edgecase given that its structured differently data-wise. + def _read_domain_escrow(self, file, seperator): + dict_data = {} + with open(file, "r", encoding="utf-8-sig") as requested_file: + reader = csv.reader(requested_file, delimiter=seperator) + for row in reader: + domain_name = row[0] + date_format = "%Y-%m-%dT%H:%M:%SZ" + # TODO - add error handling + creation_date = datetime.strptime(row[8], date_format) + expiration_date = datetime.strptime(row[10], date_format) + + dict_data[domain_name] = DomainEscrow( + domain_name, + creation_date, + expiration_date + ) + return dict_data + def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 44beab5b2..b699ee5d6 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -1,26 +1,35 @@ -from dataclasses import dataclass -@dataclass class TransitionDomainArguments: """Stores arguments for load_transition_domain""" - # Settings # - directory: str - seperator: str - limit_parse: int + + def __init__(self, **options): + # Settings # + self.directory = options.get('directory') + self.sep = options.get('sep') + self.limitParse = options.get('limitParse') + + # Filenames # + ## Adhocs ## + self.agency_adhoc_filename = options.get('agency_adhoc_filename') + self.domain_adhoc_filename = options.get('domain_adhoc_filename') + self.organization_adhoc_filename = options.get('organization_adhoc_filename') + + ## Data files ## + self.domain_additional_filename = options.get('domain_additional_filename') + self.domain_contacts_filename = options.get('domain_contacts_filename') + self.domain_statuses_filename = options.get('domain_statuses_filename') + + # Flags # + self.debug = options.get('debug') + self.resetTable = options.get('resetTable') - # Filenames # - ## Adhocs ## - agency_adhoc_filename: str - domain_adhoc_filename: str - organization_adhoc_filename: str - - ## Data files ## - domain_additional_filename: str - domain_contacts_filename: str - domain_statuses_filename: str - - # Flags # - debug: bool - reset_table: bool - load_extra: bool \ No newline at end of file + def args_extra_transition_domain(self): + return { + "agency_adhoc_filename": self.agency_adhoc_filename, + "domain_adhoc_filename": self.domain_adhoc_filename, + "organization_adhoc_filename": self.organization_adhoc_filename, + "domain_additional_filename": self.domain_additional_filename, + "directory": self.directory, + "sep": self.sep, + } \ No newline at end of file From dca5bdef72407e0309a2199157650876fd126742 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:07:22 -0600 Subject: [PATCH 13/88] Parse expiration --- .../commands/load_extra_transition_domain.py | 465 ------------------ .../commands/load_transition_domain.py | 5 +- .../commands/utility/epp_data_containers.py | 7 + ...n.py => extra_transition_domain_helper.py} | 283 ++++++++--- .../utility/transition_domain_arguments.py | 30 +- ...ransitiondomain_federal_agency_and_more.py | 16 + src/registrar/models/transition_domain.py | 12 + 7 files changed, 256 insertions(+), 562 deletions(-) delete mode 100644 src/registrar/management/commands/load_extra_transition_domain.py rename src/registrar/management/commands/utility/{extra_transition_domain.py => extra_transition_domain_helper.py} (79%) diff --git a/src/registrar/management/commands/load_extra_transition_domain.py b/src/registrar/management/commands/load_extra_transition_domain.py deleted file mode 100644 index a3fdca7e3..000000000 --- a/src/registrar/management/commands/load_extra_transition_domain.py +++ /dev/null @@ -1,465 +0,0 @@ -"""""" -import csv -import glob -import re -import logging - -import os -from typing import List -from enum import Enum -from django.core.management import BaseCommand - -from registrar.models.transition_domain import TransitionDomain -from .utility.extra_transition_domain import ExtraTransitionDomain -from .utility.epp_data_containers import ( - AgencyAdhoc, - AuthorityAdhoc, - DomainAdditionalData, - DomainTypeAdhoc, - OrganizationAdhoc, - EnumFilenames, -) - -logger = logging.getLogger(__name__) - - -class LogCode(Enum): - ERROR = 1 - WARNING = 2 - INFO = 3 - DEBUG = 4 - - -class FileTransitionLog: - """Container for storing event logs. Used to lessen - the complexity of storing multiple logs across multiple - variables. - - self.logs: dict -> { - EnumFilenames.DOMAIN_ADHOC: List[LogItem], - EnumFilenames.AGENCY_ADHOC: List[LogItem], - EnumFilenames.ORGANIZATION_ADHOC: List[LogItem], - EnumFilenames.DOMAIN_ADDITIONAL: List[LogItem], - } - """ - def __init__(self): - self.logs = { - EnumFilenames.DOMAIN_ADHOC: [], - EnumFilenames.AGENCY_ADHOC: [], - EnumFilenames.ORGANIZATION_ADHOC: [], - EnumFilenames.DOMAIN_ADDITIONAL: [], - } - - class LogItem: - """Used for storing data about logger information.""" - def __init__(self, file_type, code, message, domain_name): - self.file_type = file_type - self.code = code - self.message = message - self.domain_name = domain_name - - def add_log(self, file_type, code, message, domain_name): - """Adds a log item to self.logs - - file_type -> Which array to add to, - ex. EnumFilenames.DOMAIN_ADHOC - - code -> Log severity or other metadata, ex. LogCode.ERROR - - message -> Message to display - """ - self.logs[file_type].append(self.LogItem(file_type, code, message, domain_name)) - - def create_log_item(self, file_type, code, message, domain_name=None, add_to_list=True): - """Creates and returns an LogItem object. - - add_to_list: bool -> If enabled, add it to the logs array. - """ - log = self.LogItem(file_type, code, message, domain_name) - if not add_to_list: - return log - else: - self.logs[file_type].append(log) - return log - - def display_logs(self, file_type): - """Displays all logs in the given file_type in EnumFilenames. - Will log with the correct severity depending on code. - """ - for log in self.logs.get(file_type): - match log.code: - case LogCode.ERROR: - logger.error(log.message) - case LogCode.WARNING: - logger.warning(log.message) - case LogCode.INFO: - logger.info(log.message) - case LogCode.DEBUG: - logger.debug(log.message) - - -class Command(BaseCommand): - help = "" - filenames = EnumFilenames - parse_logs = FileTransitionLog() - - def add_arguments(self, parser): - """Add filename arguments.""" - parser.add_argument( - "--directory", default="migrationdata", help="Desired directory" - ) - parser.add_argument( - "--agency_adhoc_filename", - default=EnumFilenames.AGENCY_ADHOC.value[1], - help="Defines the filename for agency adhocs", - ) - parser.add_argument( - "--domain_additional_filename", - default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], - help="Defines the filename for additional domain data", - ) - parser.add_argument( - "--domain_adhoc_filename", - default=EnumFilenames.DOMAIN_ADHOC.value[1], - help="Defines the filename for domain type adhocs", - ) - parser.add_argument( - "--organization_adhoc_filename", - default=EnumFilenames.ORGANIZATION_ADHOC.value[1], - help="Defines the filename for domain type adhocs", - ) - parser.add_argument("--sep", default="|", help="Delimiter character") - - def handle(self, **options): - try: - self.domain_object = ExtraTransitionDomain( - agency_adhoc_filename=options["agency_adhoc_filename"], - domain_additional_filename=options["domain_additional_filename"], - domain_adhoc_filename=options["domain_adhoc_filename"], - organization_adhoc_filename=options["organization_adhoc_filename"], - directory=options["directory"], - seperator=options["sep"], - ) - self.domain_object.parse_all_files() - except Exception as err: - logger.error(f"Could not load additional data. Error: {err}") - raise err - else: - all_transition_domains = TransitionDomain.objects.all() - if not all_transition_domains.exists(): - raise Exception("No TransitionDomain objects exist.") - for transition_domain in all_transition_domains: - domain_name = transition_domain.domain_name.upper() - updated_transition_domain = transition_domain - # STEP 1: Parse organization data - updated_transition_domain = self.parse_org_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.ORGANIZATION_ADHOC) - - # STEP 2: Parse domain type data - updated_transition_domain = self.parse_domain_type_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.DOMAIN_ADHOC) - - # STEP 3: Parse agency data - updated_transition_domain = self.parse_agency_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.AGENCY_ADHOC) - - # STEP 4: Parse expiration data - TODO - updated_transition_domain = self.parse_expiration_data( - domain_name, transition_domain - ) - # self.parse_logs(EnumFilenames.EXPIRATION_DATA) - - updated_transition_domain.save() - - # TODO - Implement once Niki gets her ticket in - def parse_expiration_data(self, domain_name, transition_domain): - return transition_domain - - def parse_agency_data(self, domain_name, transition_domain) -> TransitionDomain: - if not isinstance(transition_domain, TransitionDomain): - raise ValueError("Not a valid object, must be TransitionDomain") - - info = self.get_agency_info(domain_name) - if info is None: - self.parse_logs.create_log_item( - EnumFilenames.AGENCY_ADHOC, - LogCode.ERROR, - f"Could not add federal_agency on {domain_name}, no data exists.", - domain_name - ) - return transition_domain - - agency_exists = ( - transition_domain.federal_agency is not None - and transition_domain.federal_agency.strip() != "" - ) - - if not info.active.lower() == "y": - self.parse_logs.create_log_item( - EnumFilenames.DOMAIN_ADHOC, - LogCode.ERROR, - f"Could not add inactive agency {info.agencyname} on {domain_name}", - domain_name - ) - return transition_domain - - if not info.isfederal.lower() == "y": - self.parse_logs.create_log_item( - EnumFilenames.DOMAIN_ADHOC, - LogCode.ERROR, - f"Could not add non-federal agency {info.agencyname} on {domain_name}", - domain_name - ) - return transition_domain - - transition_domain.federal_agency = info.agencyname - - # Logs if we either added to this property, - # or modified it. - self._add_or_change_message( - EnumFilenames.AGENCY_ADHOC, - "federal_agency", - transition_domain.federal_agency, - domain_name, - agency_exists - ) - - return transition_domain - - def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: - """Parses the DomainType file. - This file has definitions for organization_type and federal_agency. - Logs if - """ - if not isinstance(transition_domain, TransitionDomain): - raise ValueError("Not a valid object, must be TransitionDomain") - - info = self.get_domain_type_info(domain_name) - if info is None: - self.parse_logs.create_log_item( - EnumFilenames.DOMAIN_ADHOC, - LogCode.ERROR, - f"Could not add domain_type on {domain_name}, no data exists.", - domain_name - ) - return transition_domain - - # This data is stored as follows: FEDERAL - Judicial - # For all other records, it is stored as so: Interstate - # We can infer if it is federal or not based on this fact. - domain_type = info.domaintype.split("-") - domain_type_length = len(domain_type) - if domain_type_length < 1 or domain_type_length > 2: - raise ValueError("Found invalid data on DOMAIN_ADHOC") - - # Then, just grab the organization type. - new_organization_type = domain_type[0].strip() - - # Check if this domain_type is active or not. - # If not, we don't want to add this. - if not info.active.lower() == "y": - self.parse_logs.create_log_item( - EnumFilenames.DOMAIN_ADHOC, - LogCode.ERROR, - f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", - domain_name - ) - return transition_domain - - # Are we updating data that already exists, - # or are we adding new data in its place? - organization_type_exists = ( - transition_domain.organization_type is not None - and transition_domain.organization_type.strip() != "" - ) - federal_type_exists = ( - transition_domain.federal_type is not None - and transition_domain.federal_type.strip() != "" - ) - - # If we get two records, then we know it is federal. - # needs to be lowercase for federal type - is_federal = domain_type_length == 2 - if is_federal: - new_federal_type = domain_type[1].strip() - transition_domain.organization_type = new_organization_type - transition_domain.federal_type = new_federal_type - else: - transition_domain.organization_type = new_organization_type - transition_domain.federal_type = None - - # Logs if we either added to this property, - # or modified it. - self._add_or_change_message( - EnumFilenames.DOMAIN_ADHOC, - "organization_type", - transition_domain.organization_type, - domain_name, - organization_type_exists, - ) - - self._add_or_change_message( - EnumFilenames.DOMAIN_ADHOC, - "federal_type", - transition_domain.federal_type, - domain_name, - federal_type_exists, - ) - - return transition_domain - - def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: - if not isinstance(transition_domain, TransitionDomain): - raise ValueError("Not a valid object, must be TransitionDomain") - - org_info = self.get_org_info(domain_name) - if org_info is None: - self.parse_logs.create_log_item( - EnumFilenames.ORGANIZATION_ADHOC, - LogCode.ERROR, - f"Could not add organization_name on {domain_name}, no data exists.", - domain_name - ) - return transition_domain - - desired_property_exists = ( - transition_domain.organization_name is not None - and transition_domain.organization_name.strip() != "" - ) - - transition_domain.organization_name = org_info.orgname - - # Logs if we either added to this property, - # or modified it. - self._add_or_change_message( - EnumFilenames.ORGANIZATION_ADHOC, - "organization_name", - transition_domain.organization_name, - domain_name, - desired_property_exists, - ) - - return transition_domain - - def _add_or_change_message( - self, file_type, var_name, changed_value, domain_name, is_update=False - ): - """Creates a log instance when a property - is successfully changed on a given TransitionDomain.""" - if not is_update: - self.parse_logs.create_log_item( - file_type, - LogCode.DEBUG, - f"Added {var_name} as '{changed_value}' on {domain_name}", - domain_name - ) - else: - self.parse_logs.create_log_item( - file_type, - LogCode.INFO, - f"Updated existing {var_name} to '{changed_value}' on {domain_name}", - domain_name - ) - - # Property getters, i.e. orgid or domaintypeid - def get_org_info(self, domain_name) -> OrganizationAdhoc: - domain_info = self.get_domain_data(domain_name) - if domain_info is None: - return None - org_id = domain_info.orgid - return self.get_organization_adhoc(org_id) - - def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: - domain_info = self.get_domain_data(domain_name) - if domain_info is None: - return None - type_id = domain_info.domaintypeid - return self.get_domain_adhoc(type_id) - - def get_agency_info(self, domain_name) -> AgencyAdhoc: - domain_info = self.get_domain_data(domain_name) - if domain_info is None: - return None - type_id = domain_info.orgid - return self.get_domain_adhoc(type_id) - - def get_authority_info(self, domain_name): - domain_info = self.get_domain_data(domain_name) - if domain_info is None: - return None - type_id = domain_info.authorityid - return self.get_authority_adhoc(type_id) - - # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc - def get_domain_data(self, desired_id) -> DomainAdditionalData: - return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) - - def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: - """Grabs adhoc information for organizations. Returns an organization - adhoc object. - """ - return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) - - def get_domain_adhoc(self, desired_id) -> DomainTypeAdhoc: - """""" - return self.get_object_by_id(EnumFilenames.DOMAIN_ADHOC, desired_id) - - def get_agency_adhoc(self, desired_id) -> AgencyAdhoc: - """""" - return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) - - def get_authority_adhoc(self, desired_id) -> AuthorityAdhoc: - """""" - return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) - - def get_object_by_id(self, file_type: EnumFilenames, desired_id): - """Returns a field in a dictionary based off the type and id. - - vars: - file_type: (constant) EnumFilenames -> Which data file to target. - An example would be `EnumFilenames.DOMAIN_ADHOC`. - - desired_id: str -> Which id you want to search on. - An example would be `"12"` or `"igorville.gov"` - - Explanation: - Each data file has an associated type (file_type) for tracking purposes. - - Each file_type is a dictionary which - contains a dictionary of row[id_field]: object. - - In practice, this would look like: - - EnumFilenames.AUTHORITY_ADHOC: { - "1": AuthorityAdhoc(...), - "2": AuthorityAdhoc(...), - ... - } - - desired_id will then specify which id to grab. If we wanted "1", - then this function will return the value of id "1". - So, `AuthorityAdhoc(...)` - """ - # Grabs a dict associated with the file_type. - # For example, EnumFilenames.DOMAIN_ADDITIONAL. - desired_type = self.domain_object.file_data.get(file_type) - if desired_type is None: - self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Type {file_type} does not exist" - ) - return None - - # Grab the value given an Id within that file_type dict. - # For example, "igorville.gov". - obj = desired_type.data.get(desired_id) - if obj is None: - self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Id {desired_id} does not exist" - ) - return obj diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 3a08d70dd..496ef9c4f 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -16,7 +16,7 @@ from registrar.management.commands.utility.terminal_helper import ( ) from .utility.transition_domain_arguments import TransitionDomainArguments -from .utility.extra_transition_domain import LoadExtraTransitionDomain +from .utility.extra_transition_domain_helper import LoadExtraTransitionDomain logger = logging.getLogger(__name__) @@ -503,7 +503,7 @@ class Command(BaseCommand): # Prompt the user if they want to load additional data on the domains # TODO - add this logic into the core of this file arguments = TransitionDomainArguments(**options) - + do_parse_extra = TerminalHelper.prompt_for_execution( True, "./manage.py test", @@ -512,4 +512,3 @@ class Command(BaseCommand): if do_parse_extra: extra = LoadExtraTransitionDomain(arguments) extra_logs = extra.parse_logs.logs - diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 42dbdebd5..36ec9e19a 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -14,6 +14,7 @@ from typing import List, Optional @dataclass class AgencyAdhoc: """Defines the structure given in the AGENCY_ADHOC file""" + agencyid: Optional[int] = None agencyname: Optional[str] = None active: Optional[str] = None @@ -23,6 +24,7 @@ class AgencyAdhoc: @dataclass class DomainAdditionalData: """Defines the structure given in the DOMAIN_ADDITIONAL file""" + domainname: Optional[str] = None domaintypeid: Optional[int] = None authorityid: Optional[int] = None @@ -35,6 +37,7 @@ class DomainAdditionalData: @dataclass class DomainTypeAdhoc: """Defines the structure given in the DOMAIN_ADHOC file""" + domaintypeid: Optional[int] = None domaintype: Optional[str] = None code: Optional[str] = None @@ -44,6 +47,7 @@ class DomainTypeAdhoc: @dataclass class OrganizationAdhoc: """Defines the structure given in the ORGANIZATION_ADHOC file""" + orgid: Optional[int] = None orgname: Optional[str] = None orgstreet: Optional[str] = None @@ -56,6 +60,7 @@ class OrganizationAdhoc: @dataclass class AuthorityAdhoc: """Defines the structure given in the AUTHORITY_ADHOC file""" + authorityid: Optional[int] = None firstname: Optional[str] = None middlename: Optional[str] = None @@ -65,9 +70,11 @@ class AuthorityAdhoc: agencyid: Optional[int] = None addlinfo: Optional[List[str]] = None + @dataclass class DomainEscrow: """Defines the structure given in the DOMAIN_ESCROW file""" + domainname: Optional[str] = None creationdate: Optional[date] = None expirationdate: Optional[date] = None diff --git a/src/registrar/management/commands/utility/extra_transition_domain.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py similarity index 79% rename from src/registrar/management/commands/utility/extra_transition_domain.py rename to src/registrar/management/commands/utility/extra_transition_domain_helper.py index 3489e55c1..7bf5dab10 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -26,8 +26,10 @@ from .transition_domain_arguments import TransitionDomainArguments logger = logging.getLogger(__name__) + class LogCode(Enum): """Stores the desired log severity""" + ERROR = 1 WARNING = 2 INFO = 3 @@ -36,9 +38,9 @@ class LogCode(Enum): class FileTransitionLog: """Container for storing event logs. Used to lessen - the complexity of storing multiple logs across multiple - variables. - + the complexity of storing multiple logs across multiple + variables. + self.logs: dict -> { EnumFilenames.DOMAIN_ADHOC: List[LogItem], EnumFilenames.AGENCY_ADHOC: List[LogItem], @@ -46,16 +48,19 @@ class FileTransitionLog: EnumFilenames.DOMAIN_ADDITIONAL: List[LogItem], } """ + def __init__(self): self.logs = { EnumFilenames.DOMAIN_ADHOC: [], EnumFilenames.AGENCY_ADHOC: [], EnumFilenames.ORGANIZATION_ADHOC: [], EnumFilenames.DOMAIN_ADDITIONAL: [], + EnumFilenames.DOMAIN_ESCROW: [], } class LogItem: """Used for storing data about logger information.""" + def __init__(self, file_type, code, message, domain_name): self.file_type = file_type self.code = code @@ -65,7 +70,7 @@ class FileTransitionLog: def add_log(self, file_type, code, message, domain_name): """Adds a log item to self.logs - file_type -> Which array to add to, + file_type -> Which array to add to, ex. EnumFilenames.DOMAIN_ADHOC code -> Log severity or other metadata, ex. LogCode.ERROR @@ -74,7 +79,9 @@ class FileTransitionLog: """ self.logs[file_type].append(self.LogItem(file_type, code, message, domain_name)) - def create_log_item(self, file_type, code, message, domain_name=None, add_to_list=True): + def create_log_item( + self, file_type, code, message, domain_name=None, add_to_list=True + ): """Creates and returns an LogItem object. add_to_list: bool -> If enabled, add it to the logs array. @@ -118,7 +125,6 @@ class LoadExtraTransitionDomain: # transition domain object with that data. self.update_transition_domain_models() - def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.domain_object""" @@ -148,22 +154,62 @@ class LoadExtraTransitionDomain: ) self.parse_logs.display_logs(EnumFilenames.AGENCY_ADHOC) - # STEP 4: Parse expiration data - TODO - updated_transition_domain = self.parse_expiration_data( + # STEP 4: Parse creation and expiration data + updated_transition_domain = self.parse_creation_expiration_data( domain_name, transition_domain ) - # self.parse_logs(EnumFilenames.EXPIRATION_DATA) + self.parse_logs.display_logs(EnumFilenames.DOMAIN_ESCROW) updated_transition_domain.save() - # TODO - Implement once Niki gets her ticket in - def parse_expiration_data(self, domain_name, transition_domain): - """Grabs expiration_date from the parsed files and associates it + def parse_creation_expiration_data(self, domain_name, transition_domain): + """Grabs expiration_date from the parsed files and associates it with a transition_domain object, then returns that object.""" + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + info = self.get_domain_escrow_info(domain_name) + if info is None: + self.parse_logs.create_log_item( + EnumFilenames.DOMAIN_ESCROW, + LogCode.ERROR, + "Could not add epp_creation_date and epp_expiration_date " + f"on {domain_name}, no data exists.", + domain_name, + ) + return transition_domain + + creation_exists = ( + transition_domain.epp_creation_date is not None + ) + expiration_exists = ( + transition_domain.epp_expiration_date is not None + ) + + transition_domain.epp_creation_date = info.creationdate + transition_domain.epp_expiration_date = info.expirationdate + + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + EnumFilenames.DOMAIN_ESCROW, + "epp_creation_date", + transition_domain.epp_creation_date, + domain_name, + creation_exists, + ) + self._add_or_change_message( + EnumFilenames.DOMAIN_ESCROW, + "epp_expiration_date", + transition_domain.epp_expiration_date, + domain_name, + expiration_exists, + ) + return transition_domain def parse_agency_data(self, domain_name, transition_domain) -> TransitionDomain: - """Grabs federal_agency from the parsed files and associates it + """Grabs federal_agency from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") @@ -174,7 +220,7 @@ class LoadExtraTransitionDomain: EnumFilenames.AGENCY_ADHOC, LogCode.ERROR, f"Could not add federal_agency on {domain_name}, no data exists.", - domain_name + domain_name, ) return transition_domain @@ -188,19 +234,19 @@ class LoadExtraTransitionDomain: EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive agency {info.agencyname} on {domain_name}", - domain_name + domain_name, ) return transition_domain - + if not info.isfederal.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add non-federal agency {info.agencyname} on {domain_name}", - domain_name + domain_name, ) return transition_domain - + transition_domain.federal_agency = info.agencyname # Logs if we either added to this property, @@ -210,13 +256,15 @@ class LoadExtraTransitionDomain: "federal_agency", transition_domain.federal_agency, domain_name, - agency_exists + agency_exists, ) return transition_domain - def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: - """Grabs organization_type and federal_type from the parsed files + def parse_domain_type_data( + self, domain_name, transition_domain: TransitionDomain + ) -> TransitionDomain: + """Grabs organization_type and federal_type from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") @@ -227,7 +275,7 @@ class LoadExtraTransitionDomain: EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add domain_type on {domain_name}, no data exists.", - domain_name + domain_name, ) return transition_domain @@ -249,7 +297,7 @@ class LoadExtraTransitionDomain: EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", - domain_name + domain_name, ) return transition_domain @@ -295,8 +343,10 @@ class LoadExtraTransitionDomain: return transition_domain - def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: - """Grabs organization_name from the parsed files and associates it + def parse_org_data( + self, domain_name, transition_domain: TransitionDomain + ) -> TransitionDomain: + """Grabs organization_name from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") @@ -307,7 +357,7 @@ class LoadExtraTransitionDomain: EnumFilenames.ORGANIZATION_ADHOC, LogCode.ERROR, f"Could not add organization_name on {domain_name}, no data exists.", - domain_name + domain_name, ) return transition_domain @@ -340,19 +390,19 @@ class LoadExtraTransitionDomain: file_type, LogCode.DEBUG, f"Added {var_name} as '{changed_value}' on {domain_name}", - domain_name + domain_name, ) else: self.parse_logs.create_log_item( file_type, LogCode.INFO, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", - domain_name + domain_name, ) # Property getters, i.e. orgid or domaintypeid def get_org_info(self, domain_name) -> OrganizationAdhoc: - """Maps an id given in get_domain_data to a organization_adhoc + """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: @@ -361,7 +411,7 @@ class LoadExtraTransitionDomain: return self.get_organization_adhoc(org_id) def get_domain_type_info(self, domain_name) -> DomainTypeAdhoc: - """Maps an id given in get_domain_data to a domain_type_adhoc + """Maps an id given in get_domain_data to a domain_type_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: @@ -370,23 +420,30 @@ class LoadExtraTransitionDomain: return self.get_domain_adhoc(type_id) def get_agency_info(self, domain_name) -> AgencyAdhoc: - """Maps an id given in get_domain_data to a agency_adhoc + """Maps an id given in get_domain_data to a agency_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.orgid return self.get_domain_adhoc(type_id) - + def get_authority_info(self, domain_name): - """Maps an id given in get_domain_data to a authority_adhoc + """Maps an id given in get_domain_data to a authority_adhoc record which has its corresponding definition""" domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.authorityid return self.get_authority_adhoc(type_id) - + + def get_domain_escrow_info(self, domain_name): + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + type_id = domain_info.domainname + return self.get_domain_escrow(type_id) + # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, @@ -407,11 +464,16 @@ class LoadExtraTransitionDomain: """Grabs a corresponding row within the AGENCY_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.AGENCY_ADHOC, desired_id) - + def get_authority_adhoc(self, desired_id) -> AuthorityAdhoc: """Grabs a corresponding row within the AUTHORITY_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) + + def get_domain_escrow(self, desired_id) -> DomainEscrow: + """Grabs a corresponding row within the DOMAIN_ESCROW file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.DOMAIN_ESCROW, desired_id) def get_object_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. @@ -420,23 +482,23 @@ class LoadExtraTransitionDomain: file_type: (constant) EnumFilenames -> Which data file to target. An example would be `EnumFilenames.DOMAIN_ADHOC`. - desired_id: str -> Which id you want to search on. + desired_id: str -> Which id you want to search on. An example would be `"12"` or `"igorville.gov"` - + Explanation: Each data file has an associated type (file_type) for tracking purposes. - Each file_type is a dictionary which + Each file_type is a dictionary which contains a dictionary of row[id_field]: object. In practice, this would look like: - EnumFilenames.AUTHORITY_ADHOC: { + EnumFilenames.AUTHORITY_ADHOC: { "1": AuthorityAdhoc(...), "2": AuthorityAdhoc(...), ... } - + desired_id will then specify which id to grab. If we wanted "1", then this function will return the value of id "1". So, `AuthorityAdhoc(...)` @@ -450,7 +512,7 @@ class LoadExtraTransitionDomain: ) return None - # Grab the value given an Id within that file_type dict. + # Grab the value given an Id within that file_type dict. # For example, "igorville.gov". obj = desired_type.data.get(desired_id) if obj is None: @@ -474,9 +536,12 @@ class PatternMap: data_type: type -> Metadata about the desired type for data. id_field: str -> Defines which field should act as the id in data. - - data: dict -> The returned data. Intended to be used with data_type - to cross-reference. + This is necessary as we store lists of "data_type" in ExtraTransitionDomain as follows: + { + id_field: data_type(...), + id_field: data_type(...), + ... + } """ @@ -487,15 +552,27 @@ class PatternMap: data_type: type, id_field: str, ): + # Metadata # + ## Filename inference metadata ## self.regex = regex - self.data_type = data_type - self.id_field = id_field - self.data = {} - self.filename = filename self.could_infer = False + ## "data" object metadata ## + ### Where the data is sourced from ### + self.filename = filename + + ### What type the data is ### + self.data_type = data_type + + ### What the id should be in the holding dict ### + self.id_field = id_field + + # Object data # + self.data = {} + self.logs = [] + def try_infer_filename(self, current_file_name, default_file_name): - """Tries to match a given filename to a regex, + """Tries to match a given filename to a regex, then uses that match to generate the filename.""" # returns (filename, inferred_successfully) return self._infer_filename(self.regex, current_file_name, default_file_name) @@ -503,21 +580,36 @@ class PatternMap: def _infer_filename(self, regex: re.Pattern, matched_file_name, default_file_name): if not isinstance(regex, re.Pattern): return (self.filename, False) - + match = regex.match(matched_file_name) - + if not match: return (self.filename, False) + total_groups = len(match.groups()) + + # If no matches exist or if we have too many + # matches, then we shouldn't infer + if total_groups == 0 or total_groups > 2: + return (self.filename, False) + + # If only one match is returned, + # it means that our default matches our request + if total_groups == 1: + return (self.filename, True) + + # Otherwise, if two are returned, then + # its likely the pattern we want date = match.group(1) filename_without_date = match.group(2) - # Can the supplied self.regex do a match on the filename? + # After stripping out the date, + # do the two filenames match? can_infer = filename_without_date == default_file_name if not can_infer: return (self.filename, False) - # If so, note that and return the inferred name + # If they do, recreate the filename and return it full_filename = date + "." + filename_without_date return (full_filename, can_infer) @@ -525,6 +617,7 @@ class PatternMap: class ExtraTransitionDomain: """Helper class to aid in storing TransitionDomain data spread across multiple files.""" + strip_date_regex = re.compile(r"(?:.*\/)?(\d+)\.(.+)") def __init__( @@ -545,27 +638,57 @@ class ExtraTransitionDomain: self.seperator = sep self.all_files = glob.glob(f"{directory}*") + # Create a set with filenames as keys for quick lookup self.all_files_set = {os.path.basename(file) for file in self.all_files} - # Used for a container of values at each filename. + # Used for a container of values at each filename. # Instead of tracking each in a seperate variable, we can declare # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. pattern_map_params = [ - (EnumFilenames.AGENCY_ADHOC, agency_adhoc_filename, AgencyAdhoc, "agencyid"), - (EnumFilenames.DOMAIN_ADDITIONAL, domain_additional_filename, DomainAdditionalData, "domainname"), - (EnumFilenames.DOMAIN_ESCROW, domain_escrow_filename, DomainEscrow, "domainname"), - (EnumFilenames.DOMAIN_ADHOC, domain_adhoc_filename, DomainTypeAdhoc, "domaintypeid"), - (EnumFilenames.ORGANIZATION_ADHOC, organization_adhoc_filename, OrganizationAdhoc, "orgid"), - (EnumFilenames.AUTHORITY_ADHOC, authority_adhoc_filename, AuthorityAdhoc, "authorityid"), + ( + EnumFilenames.AGENCY_ADHOC, + agency_adhoc_filename, + AgencyAdhoc, + "agencyid", + ), + ( + EnumFilenames.DOMAIN_ADDITIONAL, + domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ESCROW, + domain_escrow_filename, + DomainEscrow, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ADHOC, + domain_adhoc_filename, + DomainTypeAdhoc, + "domaintypeid", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), + ( + EnumFilenames.AUTHORITY_ADHOC, + authority_adhoc_filename, + AuthorityAdhoc, + "authorityid", + ), ] self.file_data = self.populate_file_data(pattern_map_params) - + def populate_file_data( - self, - pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] + self, pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] ): """Populates the self.file_data field given a set of tuple params. @@ -574,10 +697,10 @@ class ExtraTransitionDomain: [ (field_type, filename, data_type, id_field), ] - + vars: file_type (EnumFilenames) -> The name of the dictionary. - Defined as a value on EnumFilenames, such as + Defined as a value on EnumFilenames, such as EnumFilenames.AGENCY_ADHOC filename (str) -> The filepath of the given @@ -635,10 +758,14 @@ class ExtraTransitionDomain: logger.error(f"Could not find file: {filename}") continue - logger.warning( - "Attempting to infer filename" - f" for file: {filename}." - ) + # Infer filename logic # + # This mode is used for development and testing only. Rather than having + # to manually define the filename each time, we can infer what the filename + # actually is. + + # Not intended for use outside of that, as it is better to assume + # the end-user wants to be specific. + logger.warning("Attempting to infer filename" f" for file: {filename}.") for filename in self.all_files: default_name = name.value[1] match = value.try_infer_filename(filename, default_name) @@ -666,7 +793,9 @@ class ExtraTransitionDomain: file_type: PatternMap = item file_type.data = {} - def parse_csv_file(self, file, seperator, dataclass_type, id_field, is_domain_escrow=False): + def parse_csv_file( + self, file, seperator, dataclass_type, id_field, is_domain_escrow=False + ): # Domain escrow is an edge case if is_domain_escrow: return self._read_domain_escrow(file, seperator) @@ -684,11 +813,9 @@ class ExtraTransitionDomain: # TODO - add error handling creation_date = datetime.strptime(row[8], date_format) expiration_date = datetime.strptime(row[10], date_format) - + dict_data[domain_name] = DomainEscrow( - domain_name, - creation_date, - expiration_date + domain_name, creation_date, expiration_date ) return dict_data @@ -704,10 +831,10 @@ class ExtraTransitionDomain: for row in reader: if None in row: print("Skipping row with None key") - #for key, value in row.items(): - #print(f"key: {key} value: {value}") + # for key, value in row.items(): + # print(f"key: {key} value: {value}") continue row_id = row[id_field] dict_data[row_id] = dataclass_type(**row) - #dict_data = {row[id_field]: dataclass_type(**row) for row in reader} - return dict_data \ No newline at end of file + # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} + return dict_data diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index b699ee5d6..459eab6d3 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -1,29 +1,27 @@ - - class TransitionDomainArguments: """Stores arguments for load_transition_domain""" def __init__(self, **options): # Settings # - self.directory = options.get('directory') - self.sep = options.get('sep') - self.limitParse = options.get('limitParse') - + self.directory = options.get("directory") + self.sep = options.get("sep") + self.limitParse = options.get("limitParse") + # Filenames # ## Adhocs ## - self.agency_adhoc_filename = options.get('agency_adhoc_filename') - self.domain_adhoc_filename = options.get('domain_adhoc_filename') - self.organization_adhoc_filename = options.get('organization_adhoc_filename') + self.agency_adhoc_filename = options.get("agency_adhoc_filename") + self.domain_adhoc_filename = options.get("domain_adhoc_filename") + self.organization_adhoc_filename = options.get("organization_adhoc_filename") ## Data files ## - self.domain_additional_filename = options.get('domain_additional_filename') - self.domain_contacts_filename = options.get('domain_contacts_filename') - self.domain_statuses_filename = options.get('domain_statuses_filename') + self.domain_additional_filename = options.get("domain_additional_filename") + self.domain_contacts_filename = options.get("domain_contacts_filename") + self.domain_statuses_filename = options.get("domain_statuses_filename") # Flags # - self.debug = options.get('debug') - self.resetTable = options.get('resetTable') - + self.debug = options.get("debug") + self.resetTable = options.get("resetTable") + def args_extra_transition_domain(self): return { "agency_adhoc_filename": self.agency_adhoc_filename, @@ -32,4 +30,4 @@ class TransitionDomainArguments: "domain_additional_filename": self.domain_additional_filename, "directory": self.directory, "sep": self.sep, - } \ No newline at end of file + } diff --git a/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py b/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py index 4de69695b..838a26cd4 100644 --- a/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py +++ b/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py @@ -38,4 +38,20 @@ class Migration(migrations.Migration): blank=True, db_index=True, help_text="Organization name", null=True ), ), + migrations.AddField( + model_name="transitiondomain", + name="epp_creation_date", + field=models.DateField( + help_text="Duplication of registry's creation date saved for ease of reporting", + null=True, + ), + ), + migrations.AddField( + model_name="transitiondomain", + name="epp_expiration_date", + field=models.DateField( + help_text="Duplication of registry's expiration date saved for ease of reporting", + null=True, + ), + ), ] diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index aca80881c..a719defe1 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -65,6 +65,18 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Federal agency", ) + epp_creation_date = models.DateField( + null=True, + help_text=( + "Duplication of registry's creation " "date saved for ease of reporting" + ), + ) + epp_expiration_date = models.DateField( + null=True, + help_text=( + "Duplication of registry's expiration " "date saved for ease of reporting" + ), + ) def __str__(self): return ( From d807209029c13e3366c3170cb0eefbfcf235df5f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:48:24 -0600 Subject: [PATCH 14/88] Fix parse expiration bug --- .../commands/utility/extra_transition_domain_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7bf5dab10..f49ccc780 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -811,8 +811,8 @@ class ExtraTransitionDomain: domain_name = row[0] date_format = "%Y-%m-%dT%H:%M:%SZ" # TODO - add error handling - creation_date = datetime.strptime(row[8], date_format) - expiration_date = datetime.strptime(row[10], date_format) + creation_date = datetime.strptime(row[7], date_format) + expiration_date = datetime.strptime(row[11], date_format) dict_data[domain_name] = DomainEscrow( domain_name, creation_date, expiration_date From ac2d2a63003226034b2bc7f7544116979378435c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:01:10 -0600 Subject: [PATCH 15/88] Cleanup --- .../commands/load_transition_domain.py | 65 ++++++++++++++++--- .../utility/extra_transition_domain_helper.py | 4 -- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 496ef9c4f..a01619325 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -79,6 +79,12 @@ class Command(BaseCommand): default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], help="Defines the filename for additional domain data", ) + parser.add_argument( + "--domain_escrow_filename", + default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], + help="Defines the filename for creation/expiration domain data", + ) + #domain_escrow_filename parser.add_argument( "--domain_adhoc_filename", default=EnumFilenames.DOMAIN_ADHOC.value[1], @@ -275,6 +281,19 @@ class Command(BaseCommand): ) TransitionDomain.objects.all().delete() + def parse_extra(self, options): + """Loads additional information for each TransitionDomain + object based off supplied files.""" + try: + # Parse data from files + extra_data = LoadExtraTransitionDomain(options) + + # Update every TransitionDomain object where applicable + extra_data.update_transition_domain_models() + except Exception as err: + logger.error("Could not load additional TransitionDomain data.") + raise err + def handle( # noqa: C901 self, domain_contacts_filename, @@ -298,6 +317,24 @@ class Command(BaseCommand): options.get("limitParse") ) # set to 0 to parse all entries + ## Variables for Additional TransitionDomain Information ## + + # Desired directory for additional TransitionDomain data + # (In the event they are stored seperately) + directory = options.get("directory") + + # Agency information + agency_adhoc_filename = options.get("agency_adhoc_filename") + # Federal agency / organization type information + domain_adhoc_filename = options.get("domain_adhoc_filename") + # Organization name information + organization_adhoc_filename = options.get("organization_adhoc_filename") + # Creation date / expiration date information + domain_escrow_filename = options.get("domain_escrow_filename") + + # Container for all additional TransitionDomain information + domain_additional_filename = options.get("domain_additional_filename") + # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) @@ -501,14 +538,26 @@ class Command(BaseCommand): self.print_summary_status_findings(domains_without_status, outlier_statuses) # Prompt the user if they want to load additional data on the domains - # TODO - add this logic into the core of this file - arguments = TransitionDomainArguments(**options) - + title = "Do you wish to load additional data for TransitionDomains?" do_parse_extra = TerminalHelper.prompt_for_execution( - True, - "./manage.py test", - "Running load_extra_transition_domains script", + system_exit_on_terminate=True, + info_to_inspect=f""" + !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING + + == == + agency_adhoc_filename: {agency_adhoc_filename} + + ==Federal agency / organization type information== + domain_adhoc_filename: {domain_adhoc_filename} + + organization_adhoc_filename: {organization_adhoc_filename} + domain_escrow_filename: {domain_escrow_filename} + domain_additional_filename: {domain_additional_filename} + == + directory: {directory} + """, + prompt_title=title, ) if do_parse_extra: - extra = LoadExtraTransitionDomain(arguments) - extra_logs = extra.parse_logs.logs + arguments = TransitionDomainArguments(**options) + self.parse_extra(arguments) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index f49ccc780..111ac7055 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -121,10 +121,6 @@ class LoadExtraTransitionDomain: self.domain_object = ExtraTransitionDomain(**arguments) self.domain_object.parse_all_files() - # Given the data we just parsed, update each - # transition domain object with that data. - self.update_transition_domain_models() - def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.domain_object""" From d88e36e5a3898f3ba4be7c3c77fb9bef99472dca Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 16:04:14 -0600 Subject: [PATCH 16/88] Migrate logs --- .../commands/utility/epp_data_containers.py | 68 +++---- .../utility/extra_transition_domain_helper.py | 180 ++++++++++++++---- 2 files changed, 178 insertions(+), 70 deletions(-) diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 36ec9e19a..6dabb78a2 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -5,7 +5,7 @@ Regarding our dataclasses: Not intended to be used as models but rather as an alternative to storing as a dictionary. By keeping it as a dataclass instead of a dictionary, we can maintain data consistency. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import date from enum import Enum from typing import List, Optional @@ -15,69 +15,69 @@ from typing import List, Optional class AgencyAdhoc: """Defines the structure given in the AGENCY_ADHOC file""" - agencyid: Optional[int] = None - agencyname: Optional[str] = None - active: Optional[str] = None - isfederal: Optional[str] = None + agencyid: Optional[int] = field(default=None, repr=True) + agencyname: Optional[str] = field(default=None, repr=True) + active: Optional[str] = field(default=None, repr=True) + isfederal: Optional[str] = field(default=None, repr=True) @dataclass class DomainAdditionalData: """Defines the structure given in the DOMAIN_ADDITIONAL file""" - domainname: Optional[str] = None - domaintypeid: Optional[int] = None - authorityid: Optional[int] = None - orgid: Optional[int] = None - securitycontactemail: Optional[str] = None - dnsseckeymonitor: Optional[str] = None - domainpurpose: Optional[str] = None + domainname: Optional[str] = field(default=None, repr=True) + domaintypeid: Optional[int] = field(default=None, repr=True) + authorityid: Optional[int] = field(default=None, repr=True) + orgid: Optional[int] = field(default=None, repr=True) + securitycontactemail: Optional[str] = field(default=None, repr=True) + dnsseckeymonitor: Optional[str] = field(default=None, repr=True) + domainpurpose: Optional[str] = field(default=None, repr=True) @dataclass class DomainTypeAdhoc: """Defines the structure given in the DOMAIN_ADHOC file""" - domaintypeid: Optional[int] = None - domaintype: Optional[str] = None - code: Optional[str] = None - active: Optional[str] = None + domaintypeid: Optional[int] = field(default=None, repr=True) + domaintype: Optional[str] = field(default=None, repr=True) + code: Optional[str] = field(default=None, repr=True) + active: Optional[str] = field(default=None, repr=True) @dataclass class OrganizationAdhoc: """Defines the structure given in the ORGANIZATION_ADHOC file""" - orgid: Optional[int] = None - orgname: Optional[str] = None - orgstreet: Optional[str] = None - orgcity: Optional[str] = None - orgstate: Optional[str] = None - orgzip: Optional[str] = None - orgcountrycode: Optional[str] = None + orgid: Optional[int] = field(default=None, repr=True) + orgname: Optional[str] = field(default=None, repr=True) + orgstreet: Optional[str] = field(default=None, repr=True) + orgcity: Optional[str] = field(default=None, repr=True) + orgstate: Optional[str] = field(default=None, repr=True) + orgzip: Optional[str] = field(default=None, repr=True) + orgcountrycode: Optional[str] = field(default=None, repr=True) @dataclass class AuthorityAdhoc: """Defines the structure given in the AUTHORITY_ADHOC file""" - authorityid: Optional[int] = None - firstname: Optional[str] = None - middlename: Optional[str] = None - lastname: Optional[str] = None - email: Optional[str] = None - phonenumber: Optional[str] = None - agencyid: Optional[int] = None - addlinfo: Optional[List[str]] = None + authorityid: Optional[int] = field(default=None, repr=True) + firstname: Optional[str] = field(default=None, repr=True) + middlename: Optional[str] = field(default=None, repr=True) + lastname: Optional[str] = field(default=None, repr=True) + email: Optional[str] = field(default=None, repr=True) + phonenumber: Optional[str] = field(default=None, repr=True) + agencyid: Optional[int] = field(default=None, repr=True) + addlinfo: Optional[List[str]] = field(default=None, repr=True) @dataclass class DomainEscrow: """Defines the structure given in the DOMAIN_ESCROW file""" - domainname: Optional[str] = None - creationdate: Optional[date] = None - expirationdate: Optional[date] = None + domainname: Optional[str] = field(default=None, repr=True) + creationdate: Optional[date] = field(default=None, repr=True) + expirationdate: Optional[date] = field(default=None, repr=True) class EnumFilenames(Enum): diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 111ac7055..97b7b2ffb 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -93,6 +93,11 @@ class FileTransitionLog: self.logs[file_type].append(log) return log + def display_all_logs(self): + """Logs every LogItem contained in this object""" + for file_type in self.logs: + self.display_logs(file_type) + def display_logs(self, file_type): """Displays all logs in the given file_type in EnumFilenames. Will log with the correct severity depending on code. @@ -107,6 +112,21 @@ class FileTransitionLog: logger.info(log.message) case LogCode.DEBUG: logger.debug(log.message) + + def clear_logs(self): + """Clears log information""" + self.logs = { + EnumFilenames.DOMAIN_ADHOC: [], + EnumFilenames.AGENCY_ADHOC: [], + EnumFilenames.ORGANIZATION_ADHOC: [], + EnumFilenames.DOMAIN_ADDITIONAL: [], + EnumFilenames.DOMAIN_ESCROW: [], + } + + def get_logs(self, file_type): + """Grabs the logs associated with + a particular file_type""" + return self.logs.get(file_type) class LoadExtraTransitionDomain: @@ -118,46 +138,90 @@ class LoadExtraTransitionDomain: arguments = options.args_extra_transition_domain() # Reads and parses migration files - self.domain_object = ExtraTransitionDomain(**arguments) - self.domain_object.parse_all_files() + self.parsed_data_container = ExtraTransitionDomain(**arguments) + self.parsed_data_container.parse_all_files() + + def create_update_model_logs(self, file_type): + """Associates runtime logs to the file_type, + such that we can determine where errors occured when + updating a TransitionDomain model.""" + logs = self.parse_logs.get_logs(file_type) + self.parsed_data_container.set_logs(file_type, logs) def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content - given in self.domain_object""" + given in self.parsed_data_container""" all_transition_domains = TransitionDomain.objects.all() if not all_transition_domains.exists(): raise Exception("No TransitionDomain objects exist.") - for transition_domain in all_transition_domains: - domain_name = transition_domain.domain_name.upper() - updated_transition_domain = transition_domain + try: + for transition_domain in all_transition_domains: + domain_name = transition_domain.domain_name.upper() + updated_transition_domain = transition_domain - # STEP 1: Parse organization data - updated_transition_domain = self.parse_org_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.ORGANIZATION_ADHOC) + # STEP 1: Parse organization data + updated_transition_domain = self.parse_org_data( + domain_name, transition_domain + ) + # Store the event logs + self.create_update_model_logs(EnumFilenames.ORGANIZATION_ADHOC) - # STEP 2: Parse domain type data - updated_transition_domain = self.parse_domain_type_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.DOMAIN_ADHOC) + # STEP 2: Parse domain type data + updated_transition_domain = self.parse_domain_type_data( + domain_name, transition_domain + ) + # Store the event logs + self.create_update_model_logs(EnumFilenames.DOMAIN_ADHOC) - # STEP 3: Parse agency data - updated_transition_domain = self.parse_agency_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.AGENCY_ADHOC) + # STEP 3: Parse agency data + updated_transition_domain = self.parse_agency_data( + domain_name, transition_domain + ) + # Store the event logs + self.create_update_model_logs(EnumFilenames.AGENCY_ADHOC) - # STEP 4: Parse creation and expiration data - updated_transition_domain = self.parse_creation_expiration_data( - domain_name, transition_domain - ) - self.parse_logs.display_logs(EnumFilenames.DOMAIN_ESCROW) + # STEP 4: Parse creation and expiration data + updated_transition_domain = self.parse_creation_expiration_data( + domain_name, transition_domain + ) + # Store the event logs + self.create_update_model_logs(EnumFilenames.DOMAIN_ADHOC) - updated_transition_domain.save() + updated_transition_domain.save() + + logger.info(f"Succesfully updated TransitionDomain {domain_name}") + self.parse_logs.clear_logs() + except Exception as err: + logger.error("Could not update all TransitionDomain objects.") + # Regardless of what occurred, log what happened. + logger.info("======Printing log stack======") + self.parse_logs.display_all_logs() + + raise err + else: + self.display_run_summary() + + def display_run_summary(self): + """Prints information about this particular run. + Organizes like data together. + """ + container = self.parsed_data_container + agency_adhoc = container.get_logs_for_type(EnumFilenames.AGENCY_ADHOC) + authority_adhoc = container.get_logs_for_type(EnumFilenames.AUTHORITY_ADHOC) + domain_additional = container.get_logs_for_type(EnumFilenames.DOMAIN_ADDITIONAL) + domain_adhoc = container.get_logs_for_type(EnumFilenames.DOMAIN_ADHOC) + domain_escrow = container.get_logs_for_type(EnumFilenames.DOMAIN_ESCROW) + organization_adhoc = container.get_logs_for_type(EnumFilenames.ORGANIZATION_ADHOC) + variable_data = [] + for file_type in self.parsed_data_container.file_data: + # Grab all logs for + logs = self.parsed_data_container.get_logs_for_type(file_type) + variable_data.append(logs) + #agency_adhoc, authority_adhoc, domain_additional, domain_adhoc, domain_escrow, organization_adhoc = variable_data + + def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it with a transition_domain object, then returns that object.""" @@ -384,14 +448,14 @@ class LoadExtraTransitionDomain: if not is_update: self.parse_logs.create_log_item( file_type, - LogCode.DEBUG, + LogCode.INFO, f"Added {var_name} as '{changed_value}' on {domain_name}", domain_name, ) else: self.parse_logs.create_log_item( file_type, - LogCode.INFO, + LogCode.WARNING, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", domain_name, ) @@ -501,7 +565,7 @@ class LoadExtraTransitionDomain: """ # Grabs a dict associated with the file_type. # For example, EnumFilenames.DOMAIN_ADDITIONAL. - desired_type = self.domain_object.file_data.get(file_type) + desired_type = self.parsed_data_container.file_data.get(file_type) if desired_type is None: self.parse_logs.create_log_item( file_type, LogCode.ERROR, f"Type {file_type} does not exist" @@ -565,7 +629,7 @@ class PatternMap: # Object data # self.data = {} - self.logs = [] + self.logs = {} def try_infer_filename(self, current_file_name, default_file_name): """Tries to match a given filename to a regex, @@ -790,16 +854,34 @@ class ExtraTransitionDomain: file_type.data = {} def parse_csv_file( - self, file, seperator, dataclass_type, id_field, is_domain_escrow=False + self, + file_type, + file, + seperator, + dataclass_type, + id_field, + is_domain_escrow=False ): # Domain escrow is an edge case if is_domain_escrow: - return self._read_domain_escrow(file, seperator) + item_to_return = self._read_domain_escrow( + file_type, + file, + seperator + ) + return item_to_return else: - return self._read_csv_file(file, seperator, dataclass_type, id_field) + item_to_return = self._read_csv_file( + file_type, + file, + seperator, + dataclass_type, + id_field + ) + return item_to_return # Domain escrow is an edgecase given that its structured differently data-wise. - def _read_domain_escrow(self, file, seperator): + def _read_domain_escrow(self, file_type, file, seperator): dict_data = {} with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.reader(requested_file, delimiter=seperator) @@ -813,9 +895,13 @@ class ExtraTransitionDomain: dict_data[domain_name] = DomainEscrow( domain_name, creation_date, expiration_date ) + + # Given this row_id, create a default log object. + # So that we can track logs on it later. + self.set_log(file_type, domain_name, []) return dict_data - def _read_csv_file(self, file, seperator, dataclass_type, id_field): + def _read_csv_file(self, file_type, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) """ @@ -832,5 +918,27 @@ class ExtraTransitionDomain: continue row_id = row[id_field] dict_data[row_id] = dataclass_type(**row) + + # Given this row_id, create a default log object. + # So that we can track logs on it later. + self.set_log(file_type, row_id, []) # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data + + # Logging logic # + def get_logs_for_type(self, file_type): + """Returns all logs for the given file_type""" + return self.file_data.get(file_type).logs + + def get_log(self, file_type, item_id): + """Returns a log of a particular id""" + logs = self.get_logs_for_type(file_type) + return logs.get(item_id) + + def set_logs_for_type(self, file_type, logs): + """Sets all logs for a given file_type""" + self.file_data[file_type] = logs + + def set_log(self, file_type, item_id, log): + """Creates a single log item under a given file_type""" + self.file_data.get(file_type)[item_id] = log From f804b0aa0c638bf5d48c7bb8fd3ff42be6336e85 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:06:54 -0600 Subject: [PATCH 17/88] Bug fixes, better logging --- .../commands/load_transition_domain.py | 10 +- .../utility/extra_transition_domain_helper.py | 231 +++++++----------- .../commands/utility/terminal_helper.py | 45 +++- 3 files changed, 143 insertions(+), 143 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index a01619325..e1b0e5308 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -543,17 +543,19 @@ class Command(BaseCommand): system_exit_on_terminate=True, info_to_inspect=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING - - == == + ==Federal agency information== agency_adhoc_filename: {agency_adhoc_filename} - ==Federal agency / organization type information== + ==Federal type / organization type information== domain_adhoc_filename: {domain_adhoc_filename} + ==Organization name information== organization_adhoc_filename: {organization_adhoc_filename} + + ==Creation date / expiration date information== domain_escrow_filename: {domain_escrow_filename} + domain_additional_filename: {domain_additional_filename} - == directory: {directory} """, prompt_title=title, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 97b7b2ffb..46d360b57 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -23,19 +23,11 @@ from .epp_data_containers import ( ) from .transition_domain_arguments import TransitionDomainArguments +from .terminal_helper import TerminalColors, TerminalHelper, LogCode logger = logging.getLogger(__name__) -class LogCode(Enum): - """Stores the desired log severity""" - - ERROR = 1 - WARNING = 2 - INFO = 3 - DEBUG = 4 - - class FileTransitionLog: """Container for storing event logs. Used to lessen the complexity of storing multiple logs across multiple @@ -50,13 +42,7 @@ class FileTransitionLog: """ def __init__(self): - self.logs = { - EnumFilenames.DOMAIN_ADHOC: [], - EnumFilenames.AGENCY_ADHOC: [], - EnumFilenames.ORGANIZATION_ADHOC: [], - EnumFilenames.DOMAIN_ADDITIONAL: [], - EnumFilenames.DOMAIN_ESCROW: [], - } + self.logs = {} class LogItem: """Used for storing data about logger information.""" @@ -70,14 +56,19 @@ class FileTransitionLog: def add_log(self, file_type, code, message, domain_name): """Adds a log item to self.logs - file_type -> Which array to add to, + file_type -> Which enum to associate with, ex. EnumFilenames.DOMAIN_ADHOC code -> Log severity or other metadata, ex. LogCode.ERROR message -> Message to display + + domain_name -> Name of the domain, i.e. "igorville.gov" """ - self.logs[file_type].append(self.LogItem(file_type, code, message, domain_name)) + log = self.LogItem(file_type, code, message, domain_name) + dict_name = (file_type, domain_name) + self._add_to_log_list(dict_name, log) + def create_log_item( self, file_type, code, message, domain_name=None, add_to_list=True @@ -89,51 +80,61 @@ class FileTransitionLog: log = self.LogItem(file_type, code, message, domain_name) if not add_to_list: return log - else: - self.logs[file_type].append(log) + + dict_name = (file_type, domain_name) + self._add_to_log_list(dict_name, log) + return log + def _add_to_log_list(self, log_name, log): + if log_name not in self.logs: + self.logs[log_name] = [log] + else: + self.logs[log_name].append(log) + def display_all_logs(self): """Logs every LogItem contained in this object""" - for file_type in self.logs: - self.display_logs(file_type) + for parent_log in self.logs: + for child_log in parent_log: + TerminalHelper.print_conditional( + True, + child_log.message, + child_log.severity + ) - def display_logs(self, file_type): - """Displays all logs in the given file_type in EnumFilenames. + def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT): + """Displays all logs of a given domain_name. Will log with the correct severity depending on code. - """ - for log in self.logs.get(file_type): - match log.code: - case LogCode.ERROR: - logger.error(log.message) - case LogCode.WARNING: - logger.warning(log.message) - case LogCode.INFO: - logger.info(log.message) - case LogCode.DEBUG: - logger.debug(log.message) - - def clear_logs(self): - """Clears log information""" - self.logs = { - EnumFilenames.DOMAIN_ADHOC: [], - EnumFilenames.AGENCY_ADHOC: [], - EnumFilenames.ORGANIZATION_ADHOC: [], - EnumFilenames.DOMAIN_ADDITIONAL: [], - EnumFilenames.DOMAIN_ESCROW: [], - } - def get_logs(self, file_type): + domain_name: str -> The domain to target, such as "igorville.gov" + + restrict_type: LogCode -> Determines if only errors of a certain + type should be displayed, such as LogCode.ERROR. + """ + for file_type in EnumFilenames: + domain_logs = self.get_logs(file_type, domain_name) + if domain_logs is None: + return None + + for log in domain_logs: + TerminalHelper.print_conditional( + restrict_type != log.code, + log.message, + log.code + ) + + def get_logs(self, file_type, domain_name): """Grabs the logs associated with - a particular file_type""" - return self.logs.get(file_type) + a particular file_type and domain_name""" + log_name = (file_type, domain_name) + return self.logs.get(log_name) class LoadExtraTransitionDomain: """Grabs additional data for TransitionDomains.""" def __init__(self, options: TransitionDomainArguments): - # Stores event logs and organizes them + # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() arguments = options.args_extra_transition_domain() @@ -141,87 +142,65 @@ class LoadExtraTransitionDomain: self.parsed_data_container = ExtraTransitionDomain(**arguments) self.parsed_data_container.parse_all_files() - def create_update_model_logs(self, file_type): - """Associates runtime logs to the file_type, - such that we can determine where errors occured when - updating a TransitionDomain model.""" - logs = self.parse_logs.get_logs(file_type) - self.parsed_data_container.set_logs(file_type, logs) - def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.parsed_data_container""" all_transition_domains = TransitionDomain.objects.all() if not all_transition_domains.exists(): - raise Exception("No TransitionDomain objects exist.") - - try: - for transition_domain in all_transition_domains: - domain_name = transition_domain.domain_name.upper() - updated_transition_domain = transition_domain + raise ValueError("No TransitionDomain objects exist.") + for transition_domain in all_transition_domains: + domain_name = transition_domain.domain_name.upper() + updated_transition_domain = transition_domain + try: # STEP 1: Parse organization data updated_transition_domain = self.parse_org_data( domain_name, transition_domain ) - # Store the event logs - self.create_update_model_logs(EnumFilenames.ORGANIZATION_ADHOC) # STEP 2: Parse domain type data updated_transition_domain = self.parse_domain_type_data( domain_name, transition_domain ) - # Store the event logs - self.create_update_model_logs(EnumFilenames.DOMAIN_ADHOC) # STEP 3: Parse agency data updated_transition_domain = self.parse_agency_data( domain_name, transition_domain ) - # Store the event logs - self.create_update_model_logs(EnumFilenames.AGENCY_ADHOC) # STEP 4: Parse creation and expiration data updated_transition_domain = self.parse_creation_expiration_data( domain_name, transition_domain ) - # Store the event logs - self.create_update_model_logs(EnumFilenames.DOMAIN_ADHOC) updated_transition_domain.save() + self.parse_logs.display_logs_by_domain_name(domain_name) + logger.info( + f"{TerminalColors.OKCYAN}" + f"Successfully updated {domain_name}" + f"{TerminalColors.ENDC}" + ) - logger.info(f"Succesfully updated TransitionDomain {domain_name}") - self.parse_logs.clear_logs() - except Exception as err: - logger.error("Could not update all TransitionDomain objects.") - # Regardless of what occurred, log what happened. - logger.info("======Printing log stack======") - self.parse_logs.display_all_logs() + # If we run into an exception on this domain, + # Just skip over it and log that it happened. + except Exception as err: + logger.debug(err) + logger.info( + f"{TerminalColors.FAIL}" + f"Exception encountered on {domain_name}. Could not update." + f"{TerminalColors.ENDC}" + ) + raise err + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED =============== + Created 123 transition domain entries, + updated 123 transition domain entries + {TerminalColors.ENDC} + """ + ) - raise err - else: - self.display_run_summary() - - def display_run_summary(self): - """Prints information about this particular run. - Organizes like data together. - """ - container = self.parsed_data_container - agency_adhoc = container.get_logs_for_type(EnumFilenames.AGENCY_ADHOC) - authority_adhoc = container.get_logs_for_type(EnumFilenames.AUTHORITY_ADHOC) - domain_additional = container.get_logs_for_type(EnumFilenames.DOMAIN_ADDITIONAL) - domain_adhoc = container.get_logs_for_type(EnumFilenames.DOMAIN_ADHOC) - domain_escrow = container.get_logs_for_type(EnumFilenames.DOMAIN_ESCROW) - organization_adhoc = container.get_logs_for_type(EnumFilenames.ORGANIZATION_ADHOC) - variable_data = [] - for file_type in self.parsed_data_container.file_data: - # Grab all logs for - logs = self.parsed_data_container.get_logs_for_type(file_type) - variable_data.append(logs) - #agency_adhoc, authority_adhoc, domain_additional, domain_adhoc, domain_escrow, organization_adhoc = variable_data - - def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it with a transition_domain object, then returns that object.""" @@ -485,8 +464,16 @@ class LoadExtraTransitionDomain: domain_info = self.get_domain_data(domain_name) if domain_info is None: return None - type_id = domain_info.orgid - return self.get_domain_adhoc(type_id) + + # The agency record is within the authority adhoc + authority_id = domain_info.authorityid + authority = self.get_authority_adhoc(authority_id) + + type_id = None + if authority is not None: + type_id = authority.agencyid + + return self.get_agency_adhoc(type_id) def get_authority_info(self, domain_name): """Maps an id given in get_domain_data to a authority_adhoc @@ -629,7 +616,6 @@ class PatternMap: # Object data # self.data = {} - self.logs = {} def try_infer_filename(self, current_file_name, default_file_name): """Tries to match a given filename to a regex, @@ -821,7 +807,7 @@ class ExtraTransitionDomain: # Infer filename logic # # This mode is used for development and testing only. Rather than having # to manually define the filename each time, we can infer what the filename - # actually is. + # actually is. # Not intended for use outside of that, as it is better to assume # the end-user wants to be specific. @@ -855,7 +841,6 @@ class ExtraTransitionDomain: def parse_csv_file( self, - file_type, file, seperator, dataclass_type, @@ -865,14 +850,12 @@ class ExtraTransitionDomain: # Domain escrow is an edge case if is_domain_escrow: item_to_return = self._read_domain_escrow( - file_type, file, seperator ) return item_to_return else: item_to_return = self._read_csv_file( - file_type, file, seperator, dataclass_type, @@ -881,7 +864,7 @@ class ExtraTransitionDomain: return item_to_return # Domain escrow is an edgecase given that its structured differently data-wise. - def _read_domain_escrow(self, file_type, file, seperator): + def _read_domain_escrow(self, file, seperator): dict_data = {} with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.reader(requested_file, delimiter=seperator) @@ -895,13 +878,9 @@ class ExtraTransitionDomain: dict_data[domain_name] = DomainEscrow( domain_name, creation_date, expiration_date ) - - # Given this row_id, create a default log object. - # So that we can track logs on it later. - self.set_log(file_type, domain_name, []) return dict_data - def _read_csv_file(self, file_type, file, seperator, dataclass_type, id_field): + def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) """ @@ -913,32 +892,12 @@ class ExtraTransitionDomain: for row in reader: if None in row: print("Skipping row with None key") - # for key, value in row.items(): - # print(f"key: {key} value: {value}") + print(dataclass_type) + for key, value in row.items(): + print(f"key: {key} value: {value}") continue row_id = row[id_field] dict_data[row_id] = dataclass_type(**row) - - # Given this row_id, create a default log object. - # So that we can track logs on it later. - self.set_log(file_type, row_id, []) # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data - # Logging logic # - def get_logs_for_type(self, file_type): - """Returns all logs for the given file_type""" - return self.file_data.get(file_type).logs - - def get_log(self, file_type, item_id): - """Returns a log of a particular id""" - logs = self.get_logs_for_type(file_type) - return logs.get(item_id) - - def set_logs_for_type(self, file_type, logs): - """Sets all logs for a given file_type""" - self.file_data[file_type] = logs - - def set_log(self, file_type, item_id, log): - """Creates a single log item under a given file_type""" - self.file_data.get(file_type)[item_id] = log diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 7327ef3bd..e8101fdbb 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -1,8 +1,25 @@ +from enum import Enum import logging import sys logger = logging.getLogger(__name__) +class LogCode(Enum): + """Stores the desired log severity + + Overview of error codes: + - 1 ERROR + - 2 WARNING + - 3 INFO + - 4 DEBUG + - 5 DEFAULT + """ + + ERROR = 1 + WARNING = 2 + INFO = 3 + DEBUG = 4 + DEFAULT = 5 class TerminalColors: """Colors for terminal outputs @@ -54,14 +71,35 @@ class TerminalHelper: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") @staticmethod - def print_conditional(print_condition: bool, print_statement: str): + def print_conditional( + print_condition: bool, + print_statement: str, + log_severity: LogCode = LogCode.DEFAULT + ): """This function reduces complexity of debug statements in other functions. It uses the logger to write the given print_statement to the - terminal if print_condition is TRUE""" + terminal if print_condition is TRUE. + + print_condition: bool -> Prints if print_condition is TRUE + + print_statement: str -> The statement to print + + log_severity: str -> Determines the severity to log at + """ # DEBUG: if print_condition: - logger.info(print_statement) + match log_severity: + case LogCode.ERROR: + logger.error(print_statement) + case LogCode.WARNING: + logger.warning(print_statement) + case LogCode.INFO: + logger.info(print_statement) + case LogCode.DEBUG: + logger.debug(print_statement) + case _: + logger.info(print_statement) @staticmethod def prompt_for_execution( @@ -100,3 +138,4 @@ class TerminalHelper: return False return True + From 265fd83fa433de34ed1b7277454de606d77ab44b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 14:40:01 -0500 Subject: [PATCH 18/88] Agency Extractor Script created --- .../commands/agency_data_extractor.py | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 src/registrar/management/commands/agency_data_extractor.py diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py new file mode 100644 index 000000000..d6b76496e --- /dev/null +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -0,0 +1,119 @@ +import argparse +import csv +import logging + +from django.core.management import BaseCommand + +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper, +) +from registrar.models.domain_application import DomainApplication + +logger = logging.getLogger(__name__) + +# DEV SHORTCUT: +# Example command for running this script: +# docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug + +class Command(BaseCommand): + help = """Loads data for domains that are in transition + (populates transition_domain model objects).""" + + def add_arguments(self, parser): + """Add file that contains agency data""" + parser.add_argument( + "agency_data_filename", help="Data file with agency information" + ) + parser.add_argument( + "--dir", default="migrationdata", help="Desired directory" + ) + parser.add_argument("--sep", default="|", help="Delimiter character") + + parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction) + + def extract_agencies( + self, + agency_data_filepath: str, + sep: str, + debug: bool + ) -> [str]: + """Extracts all the agency names from the provided agency file""" + agency_names = [] + logger.info(f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}") + with open(agency_data_filepath, "r") as agency_data_filepath: # noqa + for row in csv.reader(agency_data_filepath, delimiter=sep): + agency_name = row[1] + TerminalHelper.print_conditional(debug, f"Checking: {agency_name}") + agency_names.append(agency_name) + logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}") + return agency_names + + def compare_lists(self, new_agency_list: [str], current_agency_list: [str], debug: bool): + """ + Compares the new agency list with the current + agency list and provides the equivalent of + an outer-join on the two (printed to the terminal) + """ + + new_agencies = [] + # 1 - Get all new agencies that we don't already have (We might want to ADD these to our list) + for agency in new_agency_list: + if agency not in current_agency_list: + new_agencies.append(agency) + TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}") + + possibly_unused_agencies = [] + # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) + for agency in current_agency_list: + if agency not in new_agency_list: + possibly_unused_agencies.append(agency) + TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") + + # Print the summary of findings + # 1 - Print the list of agencies in the NEW list, which we do not already have + # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? + new_agencies_as_string = "{}".format( + ",\n ".join(map(str, new_agencies)) + ) + possibly_unused_agencies_as_string = "{}".format( + ",\n ".join(map(str, possibly_unused_agencies)) + ) + + logger.info(f""" + {TerminalColors.OKGREEN} + ======================== SUMMARY OF FINDINGS ============================ + {len(new_agency_list)} AGENCIES WERE PROVIDED in the agency file. + {len(current_agency_list)} AGENCIES ARE CURRENTLY IN OUR SYSTEM. + + {len(new_agency_list)-len(new_agencies)} AGENCIES MATCHED + (These are agencies that are in the given agency file AND in our system already) + + {len(new_agencies)} AGENCIES TO ADD: + These agencies were in the provided agency file, but are not in our system. + {TerminalColors.YELLOW}{new_agencies_as_string} + {TerminalColors.OKGREEN} + + {len(possibly_unused_agencies)} AGENCIES TO (POSSIBLY) REMOVE: + These agencies are in our system, but not in the provided agency file: + {TerminalColors.YELLOW}{possibly_unused_agencies_as_string} + {TerminalColors.ENDC} + """) + + def handle( + self, + agency_data_filename, + **options, + ): + """Parse the agency data file.""" + + # Get all the arguments + sep = options.get("sep") + debug = options.get("debug") + dir = options.get("dir") + + agency_data_file = dir+"/"+agency_data_filename + + new_agencies = self.extract_agencies(agency_data_file, sep, debug) + existing_agencies = DomainApplication.AGENCIES + self.compare_lists(new_agencies, existing_agencies, debug) \ No newline at end of file From 72cc0d0aece48eb90be5bb582303fc0c18c669cb Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 14:59:41 -0500 Subject: [PATCH 19/88] Updated agency extractor to prevent duplicate data and to provide full list of agencies from the given file --- .../commands/agency_data_extractor.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index d6b76496e..9abbee38b 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -45,7 +45,8 @@ class Command(BaseCommand): for row in csv.reader(agency_data_filepath, delimiter=sep): agency_name = row[1] TerminalHelper.print_conditional(debug, f"Checking: {agency_name}") - agency_names.append(agency_name) + if agency_name not in agency_names: + agency_names.append(agency_name) logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}") return agency_names @@ -59,14 +60,14 @@ class Command(BaseCommand): new_agencies = [] # 1 - Get all new agencies that we don't already have (We might want to ADD these to our list) for agency in new_agency_list: - if agency not in current_agency_list: + if agency not in current_agency_list and agency not in new_agencies: new_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}") possibly_unused_agencies = [] # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) for agency in current_agency_list: - if agency not in new_agency_list: + if agency not in new_agency_list and agency not in possibly_unused_agencies: possibly_unused_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") @@ -99,7 +100,21 @@ class Command(BaseCommand): {TerminalColors.YELLOW}{possibly_unused_agencies_as_string} {TerminalColors.ENDC} """) - + + print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") + if print_full_list: + full_agency_list_as_string = "{}".format( + ",\n".join(map(str, new_agency_list)) + ) + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF AGENCIES ============================" + f"\nThese are all the agencies provided by the given agency file." + f"\n{TerminalColors.YELLOW}" + f"\n{full_agency_list_as_string}" + f"{TerminalColors.OKGREEN}" + ) + def handle( self, agency_data_filename, From 4a3dc59c37291c75992e802a7dcf752cd96077de Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 16:31:31 -0500 Subject: [PATCH 20/88] updates to agency extractor (needs 1 bug fix for column) --- .../commands/agency_data_extractor.py | 82 ++++++++++++------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index 9abbee38b..d891e5838 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -9,6 +9,7 @@ from registrar.management.commands.utility.terminal_helper import ( TerminalHelper, ) from registrar.models.domain_application import DomainApplication +from registrar.models.transition_domain import TransitionDomain logger = logging.getLogger(__name__) @@ -32,13 +33,15 @@ class Command(BaseCommand): parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction) + @staticmethod def extract_agencies( - self, agency_data_filepath: str, sep: str, debug: bool ) -> [str]: - """Extracts all the agency names from the provided agency file""" + """Extracts all the agency names from the provided + agency file (skips any duplicates) and returns those + names in an array""" agency_names = [] logger.info(f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}") with open(agency_data_filepath, "r") as agency_data_filepath: # noqa @@ -50,24 +53,27 @@ class Command(BaseCommand): logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}") return agency_names - def compare_lists(self, new_agency_list: [str], current_agency_list: [str], debug: bool): + @staticmethod + def compare_agency_lists(provided_agencies: [str], + existing_agencies: [str], + debug: bool): """ - Compares the new agency list with the current - agency list and provides the equivalent of - an outer-join on the two (printed to the terminal) + Compares new_agencies with existing_agencies and + provides the equivalent of an outer-join on the two + (printed to the terminal) """ new_agencies = [] # 1 - Get all new agencies that we don't already have (We might want to ADD these to our list) - for agency in new_agency_list: - if agency not in current_agency_list and agency not in new_agencies: + for agency in provided_agencies: + if agency not in existing_agencies and agency not in new_agencies: new_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}") possibly_unused_agencies = [] # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) - for agency in current_agency_list: - if agency not in new_agency_list and agency not in possibly_unused_agencies: + for agency in existing_agencies: + if agency not in provided_agencies and agency not in possibly_unused_agencies: possibly_unused_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") @@ -84,10 +90,10 @@ class Command(BaseCommand): logger.info(f""" {TerminalColors.OKGREEN} ======================== SUMMARY OF FINDINGS ============================ - {len(new_agency_list)} AGENCIES WERE PROVIDED in the agency file. - {len(current_agency_list)} AGENCIES ARE CURRENTLY IN OUR SYSTEM. + {len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file. + {len(existing_agencies)} AGENCIES ARE CURRENTLY IN OUR SYSTEM. - {len(new_agency_list)-len(new_agencies)} AGENCIES MATCHED + {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED (These are agencies that are in the given agency file AND in our system already) {len(new_agencies)} AGENCIES TO ADD: @@ -101,20 +107,17 @@ class Command(BaseCommand): {TerminalColors.ENDC} """) - print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") - if print_full_list: - full_agency_list_as_string = "{}".format( - ",\n".join(map(str, new_agency_list)) - ) - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF AGENCIES ============================" - f"\nThese are all the agencies provided by the given agency file." - f"\n{TerminalColors.YELLOW}" - f"\n{full_agency_list_as_string}" - f"{TerminalColors.OKGREEN}" - ) - + @staticmethod + def print_agency_list(agencies): + full_agency_list_as_string = "{}".format( + ",\n".join(map(str, agencies)) + ) + logger.info( + f"\n{TerminalColors.YELLOW}" + f"\n{full_agency_list_as_string}" + f"{TerminalColors.OKGREEN}" + ) + def handle( self, agency_data_filename, @@ -130,5 +133,26 @@ class Command(BaseCommand): agency_data_file = dir+"/"+agency_data_filename new_agencies = self.extract_agencies(agency_data_file, sep, debug) - existing_agencies = DomainApplication.AGENCIES - self.compare_lists(new_agencies, existing_agencies, debug) \ No newline at end of file + hard_coded_agencies = DomainApplication.AGENCIES + transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency') + print(transition_domain_agencies) + + # OPTION to compare the agency file to our hard-coded list + print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to check {agency_data_filename} against our hard-coded list of agencies?{TerminalColors.ENDC}") + if print_full_list: + self.compare_agency_lists(new_agencies, hard_coded_agencies, debug) + + # OPTION to compare the agency file to Transition Domains + print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}") + if print_full_list: + self.compare_agency_lists(new_agencies, transition_domain_agencies, debug) + + # OPTION to print out the full list of agencies from the agency file + print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") + if print_full_list: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF IMPORTED AGENCIES ============================" + f"\nThese are all the agencies provided by the given agency file." + ) + self.print_agency_list(new_agencies) \ No newline at end of file From e7fd7b2f1098533324b584f30fa04ab7f87072a7 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 17:18:33 -0500 Subject: [PATCH 21/88] minor update to agency extractor verbage when printing results to terminal --- src/registrar/management/commands/agency_data_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index d891e5838..ffb7c8f6a 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -91,7 +91,7 @@ class Command(BaseCommand): {TerminalColors.OKGREEN} ======================== SUMMARY OF FINDINGS ============================ {len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file. - {len(existing_agencies)} AGENCIES ARE CURRENTLY IN OUR SYSTEM. + {len(existing_agencies)} AGENCIES FOUND IN THE TARGETED SYSTEM. {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED (These are agencies that are in the given agency file AND in our system already) From 795a4ec1607d745787688c1e4d9ec2e7f76d47e0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:20:41 -0600 Subject: [PATCH 22/88] Start parsing domain information --- .../transfer_transition_domains_to_domains.py | 190 +++++++++++++++++- src/registrar/models/domain_information.py | 1 - src/registrar/views/utility/mixins.py | 6 +- 3 files changed, 191 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 3697c74e1..bbd9fca91 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -14,6 +14,9 @@ from registrar.management.commands.utility.terminal_helper import ( TerminalColors, TerminalHelper, ) +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation +from registrar.models.user import User logger = logging.getLogger(__name__) @@ -57,6 +60,23 @@ class Command(BaseCommand): """, ) + def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: + updated = False + + fields_to_update = [ + 'organization_type', + 'federal_type', + 'federal_agency', + "organization_name" + ] + defaults = {field: getattr(target, field) for field in fields_to_update} + if current != target: + current = target + DomainInformation.objects.filter(domain=current.domain).update(**defaults) + updated = True + + return updated + def update_domain_status( self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool ) -> bool: @@ -93,6 +113,8 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, + domain_information_to_create, + updated_domain_information, debug_on, ): """Prints to terminal a summary of findings from @@ -102,12 +124,18 @@ class Command(BaseCommand): total_updated_domain_entries = len(updated_domain_entries) total_domain_invitation_entries = len(domain_invitations_to_create) + total_new_domain_information_entries = len(domain_information_to_create) + total_updated_domain_information_entries = len(updated_domain_information) + logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Created {total_new_entries} domain entries, Updated {total_updated_domain_entries} domain entries + Created {total_new_domain_information_entries} domain information entries, + Updated {total_updated_domain_information_entries} domain information entries, + Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) {TerminalColors.ENDC} @@ -153,6 +181,9 @@ class Command(BaseCommand): """, ) + def try_add_domain_information(self): + pass + def try_add_domain_invitation( self, domain_email: str, associated_domain: Domain ) -> DomainInvitation | None: @@ -211,11 +242,17 @@ class Command(BaseCommand): # domains to ADD domains_to_create = [] + domain_information_to_create = [] + domain_invitations_to_create = [] # domains we UPDATED updated_domain_entries = [] + updated_domain_information = [] + # domains we SKIPPED skipped_domain_entries = [] + skipped_domain_information_entries = [] + # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 @@ -232,13 +269,17 @@ class Command(BaseCommand): transition_domain_name = transition_domain.domain_name transition_domain_status = transition_domain.status transition_domain_email = transition_domain.username + transition_domain_creation_date = transition_domain.epp_creation_date + transition_domain_expiration_date = transition_domain.epp_expiration_date # DEBUG: TerminalHelper.print_conditional( debug_on, - f"""{TerminalColors.OKCYAN} - Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} - {TerminalColors.ENDC}""", # noqa + f"{TerminalColors.OKCYAN}" + "Processing Transition Domain: " + f"{transition_domain_name}, {transition_domain_status}, {transition_domain_email}" + f", {transition_domain_creation_date}, {transition_domain_expiration_date}" + f"{TerminalColors.ENDC}", # noqa ) new_domain_invitation = None @@ -261,6 +302,11 @@ class Command(BaseCommand): update_made = self.update_domain_status( transition_domain, domain_to_update, debug_on ) + + domain_to_update.created_at = transition_domain_creation_date + domain_to_update.expiration_date = transition_domain_expiration_date + domain_to_update.save() + if update_made: # keep track of updated domains for data analysis purposes updated_domain_entries.append(transition_domain.domain_name) @@ -339,8 +385,12 @@ class Command(BaseCommand): else: # no matching entry, make one new_domain = Domain( - name=transition_domain_name, state=transition_domain_status + name=transition_domain_name, + state=transition_domain_status, + expiration_date=transition_domain_expiration_date, ) + + domains_to_create.append(new_domain) # DEBUG: TerminalHelper.print_conditional( @@ -378,6 +428,136 @@ class Command(BaseCommand): break Domain.objects.bulk_create(domains_to_create) + + for transition_domain in TransitionDomain.objects.all(): + transition_domain_name = transition_domain.domain_name + + # Create associated domain information objects + domain_data = Domain.objects.filter(name=transition_domain.domain_name) + if not domain_data.exists(): + raise ValueError("No domain exists") + + domain = domain_data.get() + + org_type = transition_domain.organization_type + fed_type = transition_domain.federal_type + fed_agency = transition_domain.federal_agency + + + + valid_org_type = org_type in [choice_value for choice_value, _ in DomainApplication.OrganizationChoices.choices] + valid_fed_type = fed_type in [choice_value for choice_value, _ in DomainApplication.BranchChoices.choices] + valid_fed_agency = fed_agency in DomainApplication.AGENCIES + + default_creator, _ = User.objects.get_or_create(username="System") + + new_domain_info_data = { + 'domain': domain, + 'organization_name': transition_domain.organization_name, + "creator": default_creator, + } + + new_domain_info_data['federal_type'] = None + for item in DomainApplication.BranchChoices.choices: + print(f"it is this: {item}") + name, _ = item + if fed_type is not None and fed_type.lower() == name: + new_domain_info_data['federal_type'] = item + + new_domain_info_data['organization_type'] = org_type + new_domain_info_data['federal_agency'] = fed_agency + if valid_org_type: + new_domain_info_data['organization_type'] = org_type + else: + logger.debug(f"No org type found on {domain.name}") + + if valid_fed_type: + new_domain_info_data['federal_type'] = fed_type + else: + logger.debug(f"No federal type found on {domain.name}") + + if valid_fed_agency: + new_domain_info_data['federal_agency'] = fed_agency + else: + logger.debug(f"No federal agency found on {domain.name}") + + new_domain_info = DomainInformation(**new_domain_info_data) + + domain_information_exists = DomainInformation.objects.filter(domain=domain).exists() + + if domain_information_exists: + try: + # get the existing domain information object + domain_info_to_update = DomainInformation.objects.get(domain=domain) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + > Found existing entry in Domain Information table for: {transition_domain_name} + {TerminalColors.ENDC}""", # noqa + ) + + # for existing entry, update the status to + # the transition domain status + update_made = self.update_domain_information( + domain_info_to_update, new_domain_info, debug_on + ) + if update_made: + # keep track of updated domains for data analysis purposes + updated_domain_information.append(transition_domain.domain_name) + except DomainInformation.MultipleObjectsReturned: + # This exception was thrown once before during testing. + # While the circumstances that led to corrupt data in + # the domain table was a freak accident, and the possibility of it + # happening again is safe-guarded by a key constraint, + # better to keep an eye out for it since it would require + # immediate attention. + logger.warning( + f""" + {TerminalColors.FAIL} + !!! ERROR: duplicate entries already exist in the + Domain Information table for the following domain: + {transition_domain_name} + + RECOMMENDATION: + This means the Domain Information table is corrupt. Please + check the Domain Information table data as there should be a key + constraint which prevents duplicate entries. + + ----------TERMINATING----------""" + ) + sys.exit() + else: + # no entry was found in the domain table + # for the given domain. Create a new entry. + + # first see if we are already adding an entry for this domain. + # The unique key constraint does not allow duplicate domain entries + # even if there are different users. + existing_domain_info_in_to_create = next( + (x for x in domain_information_to_create if x.domain.name == transition_domain_name), + None, + ) + if existing_domain_info_in_to_create is not None: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + Duplicate Detected: {transition_domain_name}. + Cannot add duplicate entry. + Violates Unique Key constraint. + {TerminalColors.ENDC}""", + ) + else: + # no matching entry, make one + domain_information_to_create.append(new_domain_info) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN} Adding domain information on: {new_domain_info.domain.name} {TerminalColors.ENDC}", # noqa + ) + + DomainInformation.objects.bulk_create(domain_information_to_create) + DomainInvitation.objects.bulk_create(domain_invitations_to_create) self.print_summary_of_findings( @@ -385,5 +565,7 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, + domain_information_to_create, + updated_domain_information, debug_on, ) diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index d2bc5c53d..377d75685 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -54,7 +54,6 @@ class DomainInformation(TimeStampedModel): blank=True, help_text="Type of Organization", ) - federally_recognized_tribe = models.BooleanField( null=True, help_text="Is the tribe federally recognized", diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 97db65505..2d65e7b02 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -99,8 +99,12 @@ class DomainPermission(PermissionsLoginMixin): requested_domain = None if DomainInformation.objects.filter(id=pk).exists(): requested_domain = DomainInformation.objects.get(id=pk) + + domain_application = requested_domain.domain_application + if not hasattr(domain_application, "status"): + return True - if requested_domain.domain_application.status not in valid_domain_statuses: + if domain_application.status not in valid_domain_statuses: return False # Valid session keys exist, From 87696d9b92c20fd58dbea11afbb6b406603c6b7a Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 21:25:19 -0500 Subject: [PATCH 23/88] Some refactoring, cleanup, and fixes so that the data loads correctly. Migrations run correctly end-to-end. But unit tests need to be updated next. --- .../transfer_transition_domains_to_domains.py | 708 +++++++++++------- .../utility/extra_transition_domain_helper.py | 5 +- 2 files changed, 426 insertions(+), 287 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index bbd9fca91..ad53c77d7 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -27,6 +27,9 @@ class Command(BaseCommand): entries for every domain we ADD (but not for domains we UPDATE)""" + # ====================================================== + # ===================== ARGUMENTS ===================== + # ====================================================== def add_arguments(self, parser): parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -36,6 +39,10 @@ class Command(BaseCommand): help="Sets max number of entries to load, set to 0 to load all entries", ) + + # ====================================================== + # ===================== PRINTING ====================== + # ====================================================== def print_debug_mode_statements( self, debug_on: bool, debug_max_entries_to_parse: int ): @@ -59,54 +66,22 @@ class Command(BaseCommand): {TerminalColors.ENDC} """, ) - - def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: - updated = False - - fields_to_update = [ - 'organization_type', - 'federal_type', - 'federal_agency', - "organization_name" - ] - defaults = {field: getattr(target, field) for field in fields_to_update} - if current != target: - current = target - DomainInformation.objects.filter(domain=current.domain).update(**defaults) - updated = True - - return updated - - def update_domain_status( - self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool - ) -> bool: - """Given a transition domain that matches an existing domain, - updates the existing domain object with that status of - the transition domain. - Returns TRUE if an update was made. FALSE if the states - matched and no update was made""" - - transition_domain_status = transition_domain.status - existing_status = target_domain.state - if transition_domain_status != existing_status: - if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: - target_domain.place_client_hold(ignoreEPP=True) - else: - target_domain.revert_client_hold(ignoreEPP=True) - target_domain.save() - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, + + def parse_limit_reached(self, + debug_max_entries_to_parse: bool, + total_rows_parsed: int + ) -> bool: + if (debug_max_entries_to_parse > 0 + and total_rows_parsed >= debug_max_entries_to_parse): + logger.info( f"""{TerminalColors.YELLOW} - >> Updated {target_domain.name} state from - '{existing_status}' to '{target_domain.state}' - (no domain invitation entry added) - {TerminalColors.ENDC}""", + ----PARSE LIMIT REACHED. HALTING PARSER.---- + {TerminalColors.ENDC} + """ ) return True return False - + def print_summary_of_findings( self, domains_to_create, @@ -181,9 +156,143 @@ class Command(BaseCommand): """, ) - def try_add_domain_information(self): - pass + # ====================================================== + # =================== DOMAIN ===================== + # ====================================================== + def update_or_create_domain(self, + transition_domain: TransitionDomain, + debug_on: bool) -> (Domain, bool): + """ Given a transition domain, either finds & updates an existing + corresponding domain, or creates a new corresponding domain in + the Domain table. + + Returns the corresponding Domain object and a boolean + that is TRUE if that Domain was newly created. + """ + + # Create some local variables to make data tracing easier + transition_domain_name = transition_domain.domain_name + transition_domain_status = transition_domain.status + transition_domain_creation_date = transition_domain.epp_creation_date + transition_domain_expiration_date = transition_domain.epp_expiration_date + + domain_exists = Domain.objects.filter(name=transition_domain_name).exists() + if domain_exists: + try: + # ----------------------- UPDATE DOMAIN ----------------------- + # ---- GET THE DOMAIN + target_domain = Domain.objects.get(name=transition_domain_name) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + > Found existing entry in Domain table for: {transition_domain_name}, {target_domain.state} + {TerminalColors.ENDC}""", # noqa + ) + + # ---- UPDATE THE DOMAIN + # update the status + update_made = self.update_domain_status( + transition_domain, target_domain, debug_on + ) + # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. + + # update dates (creation and expiration) + if transition_domain_creation_date is not None: + # TODO: added this because I ran into a situation where the created_at date was null (violated a key constraint). How do we want to handle this case? + target_domain.created_at = transition_domain_creation_date + if transition_domain_expiration_date is not None: + target_domain.expiration_date = transition_domain_expiration_date + target_domain.save() + + return (target_domain, False) + + except Domain.MultipleObjectsReturned: + # This exception was thrown once before during testing. + # While the circumstances that led to corrupt data in + # the domain table was a freak accident, and the possibility of it + # happening again is safe-guarded by a key constraint, + # better to keep an eye out for it since it would require + # immediate attention. + logger.warning( + f""" + {TerminalColors.FAIL} + !!! ERROR: duplicate entries already exist in the + Domain table for the following domain: + {transition_domain_name} + + RECOMMENDATION: + This means the Domain table is corrupt. Please + check the Domain table data as there should be a key + constraint which prevents duplicate entries. + + ----------TERMINATING----------""" + ) + sys.exit() + except TransitionNotAllowed as err: + logger.warning( + f"""{TerminalColors.FAIL} + Unable to change state for {transition_domain_name} + + RECOMMENDATION: + This indicates there might have been changes to the + Domain model which were not accounted for in this + migration script. Please check state change rules + in the Domain model and ensure we are following the + correct state transition pathways. + + INTERNAL ERROR MESSAGE: + 'TRANSITION NOT ALLOWED' exception + {err} + ----------SKIPPING----------""" + ) + return (None, False) + else: + # ----------------------- CREATE DOMAIN ----------------------- + # no matching entry, make one + target_domain = Domain( + name=transition_domain_name, + state=transition_domain_status, + expiration_date=transition_domain_expiration_date, + ) + return (target_domain, True) + + + def update_domain_status( + self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool + ) -> bool: + """Given a transition domain that matches an existing domain, + updates the existing domain object with that status of + the transition domain. + Returns TRUE if an update was made. FALSE if the states + matched and no update was made""" + + transition_domain_status = transition_domain.status + existing_status = target_domain.state + if transition_domain_status != existing_status: + if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: + target_domain.place_client_hold(ignoreEPP=True) + else: + target_domain.revert_client_hold(ignoreEPP=True) + target_domain.save() + + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + >> Updated {target_domain.name} state from + '{existing_status}' to '{target_domain.state}' + (no domain invitation entry added) + {TerminalColors.ENDC}""", + ) + return True + return False + + + # ====================================================== + # ================ DOMAIN INVITATION ================== + # ====================================================== def try_add_domain_invitation( self, domain_email: str, associated_domain: Domain ) -> DomainInvitation | None: @@ -224,6 +333,169 @@ class Command(BaseCommand): return new_domain_invitation return None + # ====================================================== + # ================ DOMAIN INFORMATION ================= + # ====================================================== + def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN}" + f"Updating: {current}" + f"{TerminalColors.ENDC}"), # noqa + ) + + updated = False + + fields_to_update = [ + 'organization_type', + 'federal_type', + 'federal_agency', + "organization_name" + ] + defaults = {field: getattr(target, field) for field in fields_to_update} + if current != target: + current = target + DomainInformation.objects.filter(domain=current.domain).update(**defaults) + updated = True + + return updated + + def try_add_domain_information(self): + pass + + def create_new_domain_info(self, + transition_domain: TransitionDomain, + domain: Domain) -> DomainInformation: + org_type = transition_domain.organization_type + fed_type = transition_domain.federal_type + fed_agency = transition_domain.federal_agency + + valid_org_type = org_type in [choice_value for choice_value, _ in DomainApplication.OrganizationChoices.choices] + valid_fed_type = fed_type in [choice_value for choice_value, _ in DomainApplication.BranchChoices.choices] + valid_fed_agency = fed_agency in DomainApplication.AGENCIES + + default_creator, _ = User.objects.get_or_create(username="System") + + new_domain_info_data = { + 'domain': domain, + 'organization_name': transition_domain.organization_name, + "creator": default_creator, + } + + new_domain_info_data['federal_type'] = None + for item in DomainApplication.BranchChoices.choices: + print(f"it is this: {item}") + name, _ = item + if fed_type is not None and fed_type.lower() == name: + new_domain_info_data['federal_type'] = item + + new_domain_info_data['organization_type'] = org_type + new_domain_info_data['federal_agency'] = fed_agency + if valid_org_type: + new_domain_info_data['organization_type'] = org_type + else: + logger.debug(f"No org type found on {domain.name}") + + if valid_fed_type: + new_domain_info_data['federal_type'] = fed_type + else: + logger.debug(f"No federal type found on {domain.name}") + + if valid_fed_agency: + new_domain_info_data['federal_agency'] = fed_agency + else: + logger.debug(f"No federal agency found on {domain.name}") + + new_domain_info = DomainInformation(**new_domain_info_data) + + # DEBUG: + TerminalHelper.print_conditional( + True, + (f"{TerminalColors.MAGENTA}" + f"Created template: {new_domain_info}" + f"{TerminalColors.ENDC}"), # noqa + ) + return new_domain_info + + def update_or_create_domain_information(self, + transition_domain: TransitionDomain, + debug_on: bool) -> (DomainInformation, bool): + + transition_domain_name = transition_domain.domain_name + + # Get associated domain + domain_data = Domain.objects.filter(name=transition_domain.domain_name) + if not domain_data.exists(): + logger.warn( + f"{TerminalColors.FAIL}" + f"WARNING: No Domain exists for:" + f"{transition_domain_name}" + f"{TerminalColors.ENDC}\n" + ) + return (None, None, False) + domain = domain_data.get() + template_domain_information = self.create_new_domain_info(transition_domain, domain) + target_domain_information = None + domain_information_exists = DomainInformation.objects.filter(domain__name=transition_domain_name).exists() + if domain_information_exists: + try: + # get the existing domain information object + target_domain_information = DomainInformation.objects.get(domain__name=transition_domain_name) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.FAIL}" + f"Found existing entry in Domain Information table for:" + f"{transition_domain_name}" + f"{TerminalColors.ENDC}"), # noqa + ) + + # for existing entry, update the status to + # the transition domain status + update_made = self.update_domain_information( + target_domain_information, template_domain_information, debug_on + ) + # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. + + return (target_domain_information, domain, False) + except DomainInformation.MultipleObjectsReturned: + # This should never happen (just like with the Domain Table). + # However, because such an error did occur in the past, + # we will watch for it in this script + logger.warning( + f""" + {TerminalColors.FAIL} + !!! ERROR: duplicate entries already exist in the + Domain Information table for the following domain: + {transition_domain_name} + + RECOMMENDATION: + This means the Domain Information table is corrupt. Please + check the Domain Information table data as there should be a key + constraint which prevents duplicate entries. + + ----------TERMINATING----------""" + ) + sys.exit() + else: + # no matching entry, make one + target_domain_information = template_domain_information + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN}" + f" Adding domain information for:" + f"{transition_domain_name}" + f"{TerminalColors.ENDC}"), + ) + return (target_domain_information, domain, True) + + + + # ====================================================== + # ===================== HANDLE ======================== + # ====================================================== def handle( self, **options, @@ -244,7 +516,6 @@ class Command(BaseCommand): domains_to_create = [] domain_information_to_create = [] - domain_invitations_to_create = [] # domains we UPDATED updated_domain_entries = [] updated_domain_information = [] @@ -253,19 +524,29 @@ class Command(BaseCommand): skipped_domain_entries = [] skipped_domain_information_entries = [] + # domain invitations to ADD + domain_invitations_to_create = [] + + # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 logger.info( - f"""{TerminalColors.OKGREEN} + f"""{TerminalColors.OKCYAN} ========================== Beginning Data Transfer ========================== {TerminalColors.ENDC}""" ) + logger.info( + f"""{TerminalColors.OKCYAN} + ========= Adding Domains and Domain Invitations ========= + {TerminalColors.ENDC}""" + ) for transition_domain in TransitionDomain.objects.all(): + # Create some local variables to make data tracing easier transition_domain_name = transition_domain.domain_name transition_domain_status = transition_domain.status transition_domain_email = transition_domain.username @@ -282,86 +563,17 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}", # noqa ) - new_domain_invitation = None - # Check for existing domain entry - domain_exists = Domain.objects.filter(name=transition_domain_name).exists() - if domain_exists: - try: - # get the existing domain - domain_to_update = Domain.objects.get(name=transition_domain_name) - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - > Found existing entry in Domain table for: {transition_domain_name}, {domain_to_update.state} - {TerminalColors.ENDC}""", # noqa - ) + # ====================================================== + # ====================== DOMAIN ======================= + target_domain, was_created = self.update_or_create_domain(transition_domain, debug_on) - # for existing entry, update the status to - # the transition domain status - update_made = self.update_domain_status( - transition_domain, domain_to_update, debug_on - ) - - domain_to_update.created_at = transition_domain_creation_date - domain_to_update.expiration_date = transition_domain_expiration_date - domain_to_update.save() - - if update_made: - # keep track of updated domains for data analysis purposes - updated_domain_entries.append(transition_domain.domain_name) - - # check if we need to add a domain invitation - # (eg. for a new user) - new_domain_invitation = self.try_add_domain_invitation( - transition_domain_email, domain_to_update - ) - - except Domain.MultipleObjectsReturned: - # This exception was thrown once before during testing. - # While the circumstances that led to corrupt data in - # the domain table was a freak accident, and the possibility of it - # happening again is safe-guarded by a key constraint, - # better to keep an eye out for it since it would require - # immediate attention. - logger.warning( - f""" - {TerminalColors.FAIL} - !!! ERROR: duplicate entries already exist in the - Domain table for the following domain: - {transition_domain_name} - - RECOMMENDATION: - This means the Domain table is corrupt. Please - check the Domain table data as there should be a key - constraint which prevents duplicate entries. - - ----------TERMINATING----------""" - ) - sys.exit() - except TransitionNotAllowed as err: - skipped_domain_entries.append(transition_domain_name) - logger.warning( - f"""{TerminalColors.FAIL} - Unable to change state for {transition_domain_name} - - RECOMMENDATION: - This indicates there might have been changes to the - Domain model which were not accounted for in this - migration script. Please check state change rules - in the Domain model and ensure we are following the - correct state transition pathways. - - INTERNAL ERROR MESSAGE: - 'TRANSITION NOT ALLOWED' exception - {err} - ----------SKIPPING----------""" - ) - else: - # no entry was found in the domain table - # for the given domain. Create a new entry. - - # first see if we are already adding an entry for this domain. + debug_string = "" + if target_domain is None: + # ---------------- SKIPPED ---------------- + skipped_domain_entries.append(transition_domain_name) + debug_string = f"skipped domain: {target_domain}" + elif was_created: + # ---------------- DUPLICATE ---------------- # The unique key constraint does not allow duplicate domain entries # even if there are different users. existing_domain_in_to_create = next( @@ -369,38 +581,33 @@ class Command(BaseCommand): None, ) if existing_domain_in_to_create is not None: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} + debug_string = f"""{TerminalColors.YELLOW} Duplicate Detected: {transition_domain_name}. Cannot add duplicate entry for another username. Violates Unique Key constraint. Checking for unique user e-mail for Domain Invitations... - {TerminalColors.ENDC}""", - ) - new_domain_invitation = self.try_add_domain_invitation( - transition_domain_email, existing_domain_in_to_create - ) + {TerminalColors.ENDC}""" else: - # no matching entry, make one - new_domain = Domain( - name=transition_domain_name, - state=transition_domain_status, - expiration_date=transition_domain_expiration_date, - ) - - - domains_to_create.append(new_domain) - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"{TerminalColors.OKCYAN} Adding domain: {new_domain} {TerminalColors.ENDC}", # noqa - ) - new_domain_invitation = self.try_add_domain_invitation( - transition_domain_email, new_domain - ) + # ---------------- CREATED ---------------- + domains_to_create.append(target_domain) + debug_string = f"created domain: {target_domain}" + elif not was_created: + # ---------------- UPDATED ---------------- + updated_domain_entries.append(transition_domain.domain_name) + debug_string = f"updated domain: {target_domain}" + + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), + ) + # ====================================================== + # ================ DOMAIN INVITATIONS ================== + new_domain_invitation = self.try_add_domain_invitation( + transition_domain_email, target_domain + ) if new_domain_invitation is None: logger.info( f"{TerminalColors.YELLOW} ! No new e-mail detected !" # noqa @@ -414,152 +621,83 @@ class Command(BaseCommand): ) domain_invitations_to_create.append(new_domain_invitation) + # ------------------ Parse limit reached? ------------------ # Check parse limit and exit loop if parse limit has been reached - if ( - debug_max_entries_to_parse > 0 - and total_rows_parsed >= debug_max_entries_to_parse - ): - logger.info( - f"""{TerminalColors.YELLOW} - ----PARSE LIMIT REACHED. HALTING PARSER.---- - {TerminalColors.ENDC} - """ - ) + if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): break + + logger.info( + f"""{TerminalColors.OKCYAN} + ========= Adding Domains Information Objects ========= + {TerminalColors.ENDC}""" + ) + Domain.objects.bulk_create(domains_to_create) + DomainInvitation.objects.bulk_create(domain_invitations_to_create) + # ====================================================== + # ================= DOMAIN INFORMATION ================= for transition_domain in TransitionDomain.objects.all(): - transition_domain_name = transition_domain.domain_name + target_domain_information, associated_domain, was_created = self.update_or_create_domain_information(transition_domain, debug_on) - # Create associated domain information objects - domain_data = Domain.objects.filter(name=transition_domain.domain_name) - if not domain_data.exists(): - raise ValueError("No domain exists") - - domain = domain_data.get() - - org_type = transition_domain.organization_type - fed_type = transition_domain.federal_type - fed_agency = transition_domain.federal_agency - - - - valid_org_type = org_type in [choice_value for choice_value, _ in DomainApplication.OrganizationChoices.choices] - valid_fed_type = fed_type in [choice_value for choice_value, _ in DomainApplication.BranchChoices.choices] - valid_fed_agency = fed_agency in DomainApplication.AGENCIES - - default_creator, _ = User.objects.get_or_create(username="System") - - new_domain_info_data = { - 'domain': domain, - 'organization_name': transition_domain.organization_name, - "creator": default_creator, - } - - new_domain_info_data['federal_type'] = None - for item in DomainApplication.BranchChoices.choices: - print(f"it is this: {item}") - name, _ = item - if fed_type is not None and fed_type.lower() == name: - new_domain_info_data['federal_type'] = item - - new_domain_info_data['organization_type'] = org_type - new_domain_info_data['federal_agency'] = fed_agency - if valid_org_type: - new_domain_info_data['organization_type'] = org_type - else: - logger.debug(f"No org type found on {domain.name}") - - if valid_fed_type: - new_domain_info_data['federal_type'] = fed_type - else: - logger.debug(f"No federal type found on {domain.name}") - - if valid_fed_agency: - new_domain_info_data['federal_agency'] = fed_agency - else: - logger.debug(f"No federal agency found on {domain.name}") - - new_domain_info = DomainInformation(**new_domain_info_data) - - domain_information_exists = DomainInformation.objects.filter(domain=domain).exists() - - if domain_information_exists: - try: - # get the existing domain information object - domain_info_to_update = DomainInformation.objects.get(domain=domain) - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - > Found existing entry in Domain Information table for: {transition_domain_name} - {TerminalColors.ENDC}""", # noqa - ) - - # for existing entry, update the status to - # the transition domain status - update_made = self.update_domain_information( - domain_info_to_update, new_domain_info, debug_on - ) - if update_made: - # keep track of updated domains for data analysis purposes - updated_domain_information.append(transition_domain.domain_name) - except DomainInformation.MultipleObjectsReturned: - # This exception was thrown once before during testing. - # While the circumstances that led to corrupt data in - # the domain table was a freak accident, and the possibility of it - # happening again is safe-guarded by a key constraint, - # better to keep an eye out for it since it would require - # immediate attention. - logger.warning( - f""" - {TerminalColors.FAIL} - !!! ERROR: duplicate entries already exist in the - Domain Information table for the following domain: - {transition_domain_name} - - RECOMMENDATION: - This means the Domain Information table is corrupt. Please - check the Domain Information table data as there should be a key - constraint which prevents duplicate entries. - - ----------TERMINATING----------""" - ) - sys.exit() - else: - # no entry was found in the domain table - # for the given domain. Create a new entry. - - # first see if we are already adding an entry for this domain. - # The unique key constraint does not allow duplicate domain entries - # even if there are different users. - existing_domain_info_in_to_create = next( - (x for x in domain_information_to_create if x.domain.name == transition_domain_name), + debug_string = "" + if target_domain_information is None: + # ---------------- SKIPPED ---------------- + skipped_domain_information_entries.append(target_domain_information) + debug_string = f"skipped domain information: {target_domain_information}" + elif was_created: + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN}" + f"Checking duplicates for: {target_domain_information}" + f"{TerminalColors.ENDC}"), # noqa + ) + # ---------------- DUPLICATE ---------------- + # The unique key constraint does not allow multiple domain + # information objects to share the same domain + existing_domain_information_in_to_create = next( + (x for x in domain_information_to_create if x.domain.name == target_domain_information.domain.name), None, ) - if existing_domain_info_in_to_create is not None: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - Duplicate Detected: {transition_domain_name}. - Cannot add duplicate entry. - Violates Unique Key constraint. - {TerminalColors.ENDC}""", - ) + # TODO: this is redundant. Currently debugging....running into unique key constraint error.... + existing_domain_info = DomainInformation.objects.filter(domain__name=target_domain_information.domain.name).exists() + if existing_domain_information_in_to_create is not None or existing_domain_info: + debug_string = f"""{TerminalColors.YELLOW} + Duplicate Detected: {domain_information_to_create}. + Cannot add duplicate Domain Information object + {TerminalColors.ENDC}""" else: - # no matching entry, make one - domain_information_to_create.append(new_domain_info) - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"{TerminalColors.OKCYAN} Adding domain information on: {new_domain_info.domain.name} {TerminalColors.ENDC}", # noqa - ) + # ---------------- CREATED ---------------- + domain_information_to_create.append(target_domain_information) + debug_string = f"created domain information: {target_domain_information}" + elif not was_created: + # ---------------- UPDATED ---------------- + updated_domain_information.append(target_domain_information) + debug_string = f"updated domain information: {target_domain_information}" + else: + debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}" + + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), + ) + # ------------------ Parse limit reached? ------------------ + # Check parse limit and exit loop if parse limit has been reached + if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): + break + + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.YELLOW}" + f"Trying to add: {domain_information_to_create}" + f"{TerminalColors.ENDC}"), + ) DomainInformation.objects.bulk_create(domain_information_to_create) - DomainInvitation.objects.bulk_create(domain_invitations_to_create) - self.print_summary_of_findings( domains_to_create, updated_domain_entries, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 46d360b57..d4af2da55 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -149,6 +149,7 @@ class LoadExtraTransitionDomain: if not all_transition_domains.exists(): raise ValueError("No TransitionDomain objects exist.") + updated_trasition_domains = [] for transition_domain in all_transition_domains: domain_name = transition_domain.domain_name.upper() updated_transition_domain = transition_domain @@ -180,6 +181,7 @@ class LoadExtraTransitionDomain: f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}" ) + updated_trasition_domains.append(updated_transition_domain) # If we run into an exception on this domain, @@ -195,8 +197,7 @@ class LoadExtraTransitionDomain: logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - Created 123 transition domain entries, - updated 123 transition domain entries + updated {len(updated_trasition_domains)} transition domain entries {TerminalColors.ENDC} """ ) From 0a313af71a2ad5b89be843dd6030e195ab1dbd90 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 3 Nov 2023 08:51:16 -0600 Subject: [PATCH 24/88] Fix org_type and fed_type --- .../transfer_transition_domains_to_domains.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index ad53c77d7..81a829ebc 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -367,12 +367,29 @@ class Command(BaseCommand): def create_new_domain_info(self, transition_domain: TransitionDomain, domain: Domain) -> DomainInformation: + org_type = transition_domain.organization_type fed_type = transition_domain.federal_type fed_agency = transition_domain.federal_agency - - valid_org_type = org_type in [choice_value for choice_value, _ in DomainApplication.OrganizationChoices.choices] - valid_fed_type = fed_type in [choice_value for choice_value, _ in DomainApplication.BranchChoices.choices] + + match org_type: + case "Federal": + org_type = ("federal", "Federal") + case "Interstate": + org_type = ("interstate", "Interstate") + case "State": + org_type = ("state_or_territory", "State or territory") + case "Tribal": + org_type = ("tribal", "Tribal") + case "County": + org_type = ("county", "County") + case "City": + org_type = ("city", "City") + case "Independent Intrastate": + org_type = ("special_district", "Special district") + + valid_org_type = org_type in [(name, value) for name, value in DomainApplication.OrganizationChoices.choices] + valid_fed_type = fed_type in [value for name, value in DomainApplication.BranchChoices.choices] valid_fed_agency = fed_agency in DomainApplication.AGENCIES default_creator, _ = User.objects.get_or_create(username="System") @@ -382,23 +399,15 @@ class Command(BaseCommand): 'organization_name': transition_domain.organization_name, "creator": default_creator, } - - new_domain_info_data['federal_type'] = None - for item in DomainApplication.BranchChoices.choices: - print(f"it is this: {item}") - name, _ = item - if fed_type is not None and fed_type.lower() == name: - new_domain_info_data['federal_type'] = item - new_domain_info_data['organization_type'] = org_type - new_domain_info_data['federal_agency'] = fed_agency if valid_org_type: - new_domain_info_data['organization_type'] = org_type + new_domain_info_data['organization_type'] = org_type[0] else: logger.debug(f"No org type found on {domain.name}") if valid_fed_type: - new_domain_info_data['federal_type'] = fed_type + new_domain_info_data['federal_type'] = fed_type.lower() + pass else: logger.debug(f"No federal type found on {domain.name}") @@ -413,7 +422,7 @@ class Command(BaseCommand): TerminalHelper.print_conditional( True, (f"{TerminalColors.MAGENTA}" - f"Created template: {new_domain_info}" + f"Created Domain Information template for: {new_domain_info}" f"{TerminalColors.ENDC}"), # noqa ) return new_domain_info @@ -485,7 +494,7 @@ class Command(BaseCommand): TerminalHelper.print_conditional( debug_on, (f"{TerminalColors.OKCYAN}" - f" Adding domain information for:" + f"Adding domain information for: " f"{transition_domain_name}" f"{TerminalColors.ENDC}"), ) @@ -585,8 +594,6 @@ class Command(BaseCommand): Duplicate Detected: {transition_domain_name}. Cannot add duplicate entry for another username. Violates Unique Key constraint. - - Checking for unique user e-mail for Domain Invitations... {TerminalColors.ENDC}""" else: # ---------------- CREATED ---------------- From 59cb3cc0dec0d58e7ac9f7840845f3b33af895de Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 3 Nov 2023 08:58:15 -0600 Subject: [PATCH 25/88] Update transfer_transition_domains_to_domains.py --- .../commands/transfer_transition_domains_to_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 81a829ebc..48b4d7844 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -689,7 +689,7 @@ class Command(BaseCommand): # DEBUG: TerminalHelper.print_conditional( debug_on, - (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), + (f"{TerminalColors.OKCYAN}{debug_string}{TerminalColors.ENDC}"), ) # ------------------ Parse limit reached? ------------------ From 501aa59c9ef7942e61b00082d2fc346a6099cf02 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 3 Nov 2023 12:45:59 -0600 Subject: [PATCH 26/88] Typo in logger --- .../management/commands/load_transition_domain.py | 7 ++++--- .../commands/transfer_transition_domains_to_domains.py | 2 +- src/registrar/views/utility/mixins.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index e1b0e5308..bbf545b47 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -142,9 +142,10 @@ class Command(BaseCommand): logger.info("Reading contacts data file %s", contacts_filename) with open(contacts_filename, "r") as contacts_file: for row in csv.reader(contacts_file, delimiter=sep): - user_id = row[0] - user_email = row[6] - user_emails_dictionary[user_id] = user_email + if row != []: + user_id = row[0] + user_email = row[6] + user_emails_dictionary[user_id] = user_email logger.info("Loaded emails for %d users", len(user_emails_dictionary)) return user_emails_dictionary diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 48b4d7844..d3f8d1aa9 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -672,7 +672,7 @@ class Command(BaseCommand): existing_domain_info = DomainInformation.objects.filter(domain__name=target_domain_information.domain.name).exists() if existing_domain_information_in_to_create is not None or existing_domain_info: debug_string = f"""{TerminalColors.YELLOW} - Duplicate Detected: {domain_information_to_create}. + Duplicate Detected: {existing_domain_information_in_to_create}. Cannot add duplicate Domain Information object {TerminalColors.ENDC}""" else: diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 2d65e7b02..9d99c48b0 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -99,7 +99,7 @@ class DomainPermission(PermissionsLoginMixin): requested_domain = None if DomainInformation.objects.filter(id=pk).exists(): requested_domain = DomainInformation.objects.get(id=pk) - + domain_application = requested_domain.domain_application if not hasattr(domain_application, "status"): return True From d9796a15afaa7d7ae8f03741fba1b88e2e02b1da Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 3 Nov 2023 14:29:12 -0500 Subject: [PATCH 27/88] updates & fixes --- .../management/commands/load_transition_domain.py | 7 +++++-- .../commands/master_domain_migrations.py | 15 ++++++++++++++- .../commands/utility/terminal_helper.py | 7 +++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index e1b0e5308..39698d826 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -66,6 +66,8 @@ class Command(BaseCommand): ) # TODO - Narrow this down + # TODO - this isn't pulling in the directory from the master script. Needs to be corrected @Nicolle - todo + # default="/app/tmp" parser.add_argument( "--directory", default="migrationdata", help="Desired directory" ) @@ -291,8 +293,9 @@ class Command(BaseCommand): # Update every TransitionDomain object where applicable extra_data.update_transition_domain_models() except Exception as err: - logger.error("Could not load additional TransitionDomain data.") + logger.error(f"Could not load additional TransitionDomain data. {err}") raise err + # TODO: handle this better...needs more logging def handle( # noqa: C901 self, @@ -540,7 +543,7 @@ class Command(BaseCommand): # Prompt the user if they want to load additional data on the domains title = "Do you wish to load additional data for TransitionDomains?" do_parse_extra = TerminalHelper.prompt_for_execution( - system_exit_on_terminate=True, + system_exit_on_terminate=False, info_to_inspect=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING ==Federal agency information== diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 3f4841129..faba8a037 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -28,8 +28,12 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): - help = """ """ + help = """ """ # TODO: update this! + + # ====================================================== + # ================== ARGUMENTS =================== + # ====================================================== def add_arguments(self, parser): """ OPTIONAL ARGUMENTS: @@ -134,6 +138,11 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) + + # ====================================================== + # =============== DATA ANALYSIS ================== + # ====================================================== + def compare_tables(self, debug_on: bool): """Does a diff between the transition_domain and the following tables: domain, domain_information and the domain_invitation. @@ -252,6 +261,10 @@ class Command(BaseCommand): """ ) + + # ====================================================== + # ================= MIGRATIONS =================== + # ====================================================== def run_load_transition_domain_script( self, file_location: str, diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index e8101fdbb..78ffb525f 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -70,6 +70,13 @@ class TerminalHelper: else: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + # @staticmethod + # def array_as_string(array_to_convert: []) -> str: + # array_as_string = "{}".format( + # ", ".join(map(str, array_to_convert)) + # ) + # return array_as_string + @staticmethod def print_conditional( print_condition: bool, From bca2a961ad730f40048566f24fccd017a10976cd Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 3 Nov 2023 15:10:46 -0500 Subject: [PATCH 28/88] Updates to agency data extractor --- .../commands/agency_data_extractor.py | 71 ++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index ffb7c8f6a..3b7931f8d 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -32,6 +32,7 @@ class Command(BaseCommand): parser.add_argument("--sep", default="|", help="Delimiter character") parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction) + parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) @staticmethod def extract_agencies( @@ -108,7 +109,7 @@ class Command(BaseCommand): """) @staticmethod - def print_agency_list(agencies): + def print_agency_list(agencies, filename): full_agency_list_as_string = "{}".format( ",\n".join(map(str, agencies)) ) @@ -117,6 +118,9 @@ class Command(BaseCommand): f"\n{full_agency_list_as_string}" f"{TerminalColors.OKGREEN}" ) + logger.info(f"{TerminalColors.MAGENTA}Writing to file...{TerminalColors.ENDC}") + with open(f"tmp/[{filename}].txt", "w+") as f: + f.write(full_agency_list_as_string) def handle( self, @@ -128,31 +132,80 @@ class Command(BaseCommand): # Get all the arguments sep = options.get("sep") debug = options.get("debug") + prompt = options.get("prompt") dir = options.get("dir") agency_data_file = dir+"/"+agency_data_filename new_agencies = self.extract_agencies(agency_data_file, sep, debug) hard_coded_agencies = DomainApplication.AGENCIES - transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency') + merged_agencies = new_agencies + for agency in hard_coded_agencies: + if agency not in merged_agencies: + merged_agencies.append(agency) + + transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency').distinct() print(transition_domain_agencies) + prompt_successful = False + # OPTION to compare the agency file to our hard-coded list - print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to check {agency_data_filename} against our hard-coded list of agencies?{TerminalColors.ENDC}") - if print_full_list: + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}") + if prompt_successful or not prompt: self.compare_agency_lists(new_agencies, hard_coded_agencies, debug) # OPTION to compare the agency file to Transition Domains - print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}") - if print_full_list: + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}") + if prompt_successful or not prompt: self.compare_agency_lists(new_agencies, transition_domain_agencies, debug) # OPTION to print out the full list of agencies from the agency file - print_full_list = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") - if print_full_list: + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") + if prompt_successful or not prompt: logger.info( f"\n{TerminalColors.OKGREEN}" f"\n======================== FULL LIST OF IMPORTED AGENCIES ============================" f"\nThese are all the agencies provided by the given agency file." + f"\n\n{len(new_agencies)} TOTAL\n\n" ) - self.print_agency_list(new_agencies) \ No newline at end of file + self.print_agency_list(new_agencies, "Imported_Agencies") + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF AGENCIES IN DROPDOWN ============================" + f"\nThese are all the agencies hard-coded in our system for the dropdown list." + f"\n\n{len(hard_coded_agencies)} TOTAL\n\n" + ) + self.print_agency_list(hard_coded_agencies, "Dropdown_Agencies") + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF AGENCIES IN TRANSITION DOMAIN ============================" + f"\nThese are all the agencies in the Transition Domains table." + f"\n\n{len(transition_domain_agencies)} TOTAL\n\n" + ) + self.print_agency_list(transition_domain_agencies, "Transition_Domain_Agencies") + + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== MERGED LISTS (dropdown + agency file) ============================" + f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." + f"\n\n{len(merged_agencies)} TOTAL\n\n" + ) + self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") \ No newline at end of file From 715f14d266119487293e5accf6a7ce9440511ca7 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 09:33:52 -0700 Subject: [PATCH 29/88] Adding some test data --- .../tests/data/test_agency_adhoc.txt | 51 +++++++++++++++++++ .../tests/data/test_authority_adhoc.txt | 51 +++++++++++++++++++ .../tests/data/test_domain_additional.txt | 51 +++++++++++++++++++ .../tests/data/test_organization_adhoc_.txt | 51 +++++++++++++++++++ 4 files changed, 204 insertions(+) create mode 100644 src/registrar/tests/data/test_agency_adhoc.txt create mode 100644 src/registrar/tests/data/test_authority_adhoc.txt create mode 100644 src/registrar/tests/data/test_domain_additional.txt create mode 100644 src/registrar/tests/data/test_organization_adhoc_.txt diff --git a/src/registrar/tests/data/test_agency_adhoc.txt b/src/registrar/tests/data/test_agency_adhoc.txt new file mode 100644 index 000000000..1b0108aee --- /dev/null +++ b/src/registrar/tests/data/test_agency_adhoc.txt @@ -0,0 +1,51 @@ +agencyid|agencyname|active|isfederal +1|Thoughtstorm|N|Y +2|Minyx|Y|N +3|Demivee|N|Y +4|InnoZ|Y|Y +5|Jayo|N|Y +6|Avaveo|N|Y +7|Thoughtmix|N|N +8|Photofeed|Y|N +9|Gabtype|N|Y +10|Youfeed|Y|N +11|Kwimbee|N|Y +12|Linkbridge|N|Y +13|Wikibox|Y|Y +14|Browsezoom|Y|Y +15|Zoozzy|Y|N +16|Mydeo|Y|Y +17|Chatterpoint|N|Y +18|Roodel|N|Y +19|Mybuzz|N|N +20|Thoughtmix|Y|Y +21|Brainlounge|N|Y +22|Quinu|Y|Y +23|Gigaclub|Y|N +24|Eare|Y|N +25|Einti|Y|N +26|Einti|Y|Y +27|Skidoo|Y|Y +28|Aibox|N|Y +29|Zoozzy|Y|Y +30|Centimia|Y|N +31|Einti|N|Y +32|Devcast|Y|N +33|Tagfeed|N|Y +34|Jabbersphere|Y|Y +35|Yamia|Y|Y +36|Fliptune|Y|N +37|Oloo|N|Y +38|Yozio|N|N +39|Brainsphere|Y|Y +40|Geba|Y|Y +41|Wikibox|N|Y +42|Topdrive|Y|Y +43|Lazz|N|N +44|Rooxo|Y|Y +45|Jetwire|N|N +46|Zoomzone|Y|Y +47|Thoughtbridge|Y|N +48|Pixope|Y|N +49|Quatz|N|N +50|Eare|N|Y diff --git a/src/registrar/tests/data/test_authority_adhoc.txt b/src/registrar/tests/data/test_authority_adhoc.txt new file mode 100644 index 000000000..007930ccc --- /dev/null +++ b/src/registrar/tests/data/test_authority_adhoc.txt @@ -0,0 +1,51 @@ +authorityid|firstname|middlename|lastname|email|phonenumber|agencyid|addlinfo +1|Gregoor||Kalinke|gkalinke0@indiegogo.com|(773) 1725515|1|Asparagus - Mexican +2|Fayre||Filippozzi|ffilippozzi1@hugedomains.com|(357) 4874280|2|Steampan - Foil +3|Gabey||Lightbody|glightbody2@fc2.com|(332) 8165691|3|Soup - Campbells, Minestrone +4|Seline||Tower|stower3@answers.com|(151) 5396028|4|Kiwi Gold Zespri +5|Rahel||Bruhnsen|rbruhnsen4@google.de|(221) 9271443|5|Miso - Soy Bean Paste +6|Barny||Hopfer|bhopfer5@geocities.jp|(785) 6558321|6|Rice - Jasmine Sented +7|Egan|Loris, slender|Tanslie|etanslie6@auda.org.au|(867) 8852523|7|Compound - Raspberry +8|Meg|Lesser mouse lemur|McLucky|mmclucky7@sciencedirect.com|(207) 4569199|8|Potatoes - Yukon Gold 5 Oz +9|Roarke|American alligator|Thackwray|rthackwray8@un.org|(227) 1557126|9|Schnappes - Peach, Walkers +10|Matteo|Tern, royal|Pancost|mpancost9@typepad.com|(425) 7967200|10|Tortillas - Flour, 12 +11|Wilhelmine||Hradsky|whradskya@tmall.com|(229) 6901308|11|Pail For Lid 1537 +12|Merrile||Dalyiel|mdalyielb@pagesperso-orange.fr|(370) 8234182|12|Table Cloth 90x90 Colour +13|Addy||Jimenez|ajimenezc@angelfire.com|(533) 1368420|13|Wanton Wrap +14|Florella||Tellwright|ftellwrightd@house.gov|(813) 6121895|14|Water - Tonic +15|Jacenta||Flewitt|jflewitte@goo.ne.jp|(884) 4307761|15|Veal - Insides, Grains +16|Nady|Baboon, gelada|Senten|nsentenf@yandex.ru|(996) 6939643|16|Soho Lychee Liqueur +17|Mano|Common palm civet|D'Cruze|mdcruzeg@ox.ac.uk|(114) 9154228|17|Goat - Whole Cut +18|Tadio||Walsh|twalshh@wunderground.com|(490) 6376756|18|Pomello +19|Carey||Boler|cboleri@google.co.jp|(439) 9984218|19|Wasabi Powder +20|Aldus||Denington|adeningtonj@npr.org|(443) 5882262|20|Devonshire Cream +21|Herculie|Horned lark|Delooze|hdeloozek@jimdo.com|(842) 7054442|21|Higashimaru Usukuchi Soy +22|Gertrud||Rosenzwig|grosenzwigl@bloglovin.com|(878) 8890041|22|Apples - Sliced / Wedge +23|Cece||Grimme|cgrimmem@senate.gov|(802) 2135321|23|Capers - Ox Eye Daisy +24|Leann|Bird, red-billed tropic|McGrey|lmcgreyn@cbsnews.com|(681) 8510458|24|Liqueur - Melon +25|Rosabelle|Turtle, eastern box|Rennels|rrennelso@vinaora.com|(447) 9158723|25|Turnip - White, Organic +26|Clay|Duck, white-faced whistling|Acland|caclandp@oracle.com|(387) 6213827|26|Veal - Tenderloin, Untrimmed +27|Devland||Hugland|dhuglandq@ning.com|(391) 8516099|27|Veal - Eye Of Round +28|Cris|Bleu, red-cheeked cordon|Morville|cmorviller@archive.org|(751) 2295767|28|Pie Shells 10 +29|Jehu||Probetts|jprobettss@mac.com|(719) 3208086|29|Pasta - Ravioli +30|Kamilah||Bartalin|kbartalint@devhub.com|(889) 5426094|30|Oranges +31|Katherine|Crane, brolga|Degoe|kdegoeu@weebly.com|(815) 7408114|31|Oranges - Navel, 72 +32|Kassey||Riba|kribav@soup.io|(267) 4032421|32|Sauce - Vodka Blush +33|Marcello||Woodman|mwoodmanw@dell.com|(869) 5497448|33|Potatoes - Instant, Mashed +34|Marie-jeanne||Yo|myox@nymag.com|(414) 2722319|34|Cafe Royale +35|Zerk||Morland|zmorlandy@xing.com|(684) 4155779|35|Muskox - French Rack +36|Rene|Darwin ground finch (unidentified)|Booker|rbookerz@tmall.com|(727) 7131800|36|Onions - Red +37|Romain||Kinnie|rkinnie10@cdc.gov|(967) 3799924|37|Lettuce - Spring Mix +38|Fredra||Denisot|fdenisot11@google.es|(986) 9408987|38|Vinegar - Raspberry +39|Ania||Djurdjevic|adjurdjevic12@wikispaces.com|(854) 8149676|39|Crawfish +40|Gretal|Red-cheeked cordon bleu|Winson|gwinson13@istockphoto.com|(407) 3343406|40|Chips - Miss Vickies +41|Ibby|Squirrel, antelope ground|Bediss|ibediss14@webs.com|(517) 5564511|41|Sobe - Orange Carrot +42|Kingsley||Sawl|ksawl15@reuters.com|(994) 8049936|42|Water - Mineral, Carbonated +43|Syd|Lapwing, southern|Valente|svalente16@whitehouse.gov|(442) 8663735|43|Pork - Sausage, Medium +44|Jsandye||Maylin|jmaylin17@archive.org|(560) 9571021|44|Muffin Hinge Container 6 +45|Beilul||Sedworth|bsedworth18@noaa.gov|(590) 1848805|45|Coriander - Seed +46|Dudley||Note|dnote19@unblog.fr|(674) 5901607|46|Apple - Custard +47|Berte||Forsdicke|bforsdicke1a@theguardian.com|(680) 4006701|47|Paste - Black Olive +48|Gwendolen|Magpie, australian|Drawmer|gdrawmer1b@nba.com|(414) 1746171|48|Extract - Almond +49|Ade||Wilkes|awilkes1c@google.es|(340) 1804264|49|Onions - Pearl +50|Nils||Burnard|nburnard1d@tuttocitta.it|(729) 4332944|50|Basil - Pesto Sauce diff --git a/src/registrar/tests/data/test_domain_additional.txt b/src/registrar/tests/data/test_domain_additional.txt new file mode 100644 index 000000000..15c90326e --- /dev/null +++ b/src/registrar/tests/data/test_domain_additional.txt @@ -0,0 +1,51 @@ +domainname|domaintypeid|authorityid|orgid|securitycontactemail|dnsseckeymonitor|domainpurpose +indiegogo.com|1|1|1|ggennrich0@utexas.edu|N|Praesent id massa id nisl venenatis lacinia. +gravatar.com|2|2|2|lrome1@uol.com.br|Y|In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. +multiply.com|3|3|3|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. +free.fr|4|4|4|plarderot3@t.co|Y|Morbi quis tortor id nulla ultrices aliquet. Maecenas leo odio, condimentum id, luctus nec, molestie sed, justo. Pellentesque viverra pede ac diam. +washingtonpost.com|5|5|5|dchaim4@yahoo.co.jp|N|Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. Quisque porta volutpat erat. +simplemachines.org|6|6|6|iyaxley5@slashdot.org|N|Phasellus in felis. +ted.com|7|7|7|ktresler6@netscape.com|N|Integer pede justo, lacinia eget, tincidunt eget, tempus vel, pede. Morbi porttitor lorem id ligula. Suspendisse ornare consequat lectus. +diigo.com|8|8|8|dfiggures7@opera.com|Y| +cafepress.com|9|9|9|npowys8@psu.edu|N|Integer a nibh. In quis justo. +slashdot.org|10|10|10||N|Curabitur at ipsum ac tellus semper interdum. Mauris ullamcorper purus sit amet nulla. Quisque arcu libero, rutrum ac, lobortis vel, dapibus at, diam. +shop-pro.jp|11|11|11|laspina@e-recht24.de|N|Aliquam non mauris. Morbi non lectus. +nature.com|12|12|12|wlanonb@bloomberg.com|Y|Praesent id massa id nisl venenatis lacinia. +comsenz.com|13|13|13|jhalloranc@wiley.com|Y|Morbi ut odio. +ed.gov|14|14|14|lpatemand@mit.edu|N|Donec odio justo, sollicitudin ut, suscipit a, feugiat et, eros. Vestibulum ac est lacinia nisi venenatis tristique. +webnode.com|15|15|15|nhirthe@storify.com|Y|Mauris lacinia sapien quis libero. Nullam sit amet turpis elementum ligula vehicula consequat. Morbi a ipsum. +nature.com|16|16|16|nnussiif@yale.edu|N|Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. +prweb.com|17|17|17||Y|Morbi ut odio. Cras mi pede, malesuada in, imperdiet et, commodo vulputate, justo. In blandit ultrices enim. +economist.com|18|18|18|klarkbyh@usnews.com|N|Nulla suscipit ligula in lacus. +i2i.jp|19|19|19|rharrisi@foxnews.com|N|Ut at dolor quis odio consequat varius. Integer ac leo. +discuz.net|20|20|20|bmeaj@miibeian.gov.cn|N|Morbi quis tortor id nulla ultrices aliquet. +google.de|21|21|21|ubakesefk@4shared.com|Y|Etiam vel augue. Vestibulum rutrum rutrum neque. +trellian.com|22|22|22|ekleimtl@amazon.co.jp|N|Duis bibendum. Morbi non quam nec dui luctus rutrum. Nulla tellus. +hhs.gov|23|23|23|mbulmanm@nymag.com|Y|Vestibulum rutrum rutrum neque. +whitehouse.gov|24|24|24|cstudden@fc2.com|N|Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus. Phasellus in felis. +cbc.ca|25|25|25|kbuskeo@shutterfly.com|Y| +prweb.com|26|26|26|hcoldbathp@reddit.com|N|Nullam varius. Nulla facilisi. Cras non velit nec nisi vulputate nonummy. +wunderground.com|27|27|27|nmessinghamq@macromedia.com|Y|Aliquam erat volutpat. In congue. +netlog.com|28|28|28|rthawr@ow.ly|Y|Nunc purus. Phasellus in felis. +biblegateway.com|29|29|29|whurlstons@github.com|Y| +istockphoto.com|30|30|30|mjiroutekt@un.org|N|Pellentesque eget nunc. Donec quis orci eget orci vehicula condimentum. Curabitur in libero ut massa volutpat convallis. +nyu.edu|31|31|31|kpatmoreu@hhs.gov|N|Morbi odio odio, elementum eu, interdum eu, tincidunt in, leo. Maecenas pulvinar lobortis est. Phasellus sit amet erat. +github.io|32|32|32|tgaberv@businessweek.com|N|Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. +globo.com|33|33|33|tmidlarw@google.com.br|N|Maecenas rhoncus aliquam lacus. +constantcontact.com|34|34|34|plaverenzx@cdbaby.com|Y|Aenean lectus. Pellentesque eget nunc. Donec quis orci eget orci vehicula condimentum. +howstuffworks.com|35|35|35|agermainy@bloomberg.com|N|Nullam molestie nibh in lectus. Pellentesque at nulla. Suspendisse potenti. +stanford.edu|36|36|36|ndabornz@smh.com.au|Y|Cras non velit nec nisi vulputate nonummy. Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. +csmonitor.com|37|37|37||Y|Nulla justo. Aliquam quis turpis eget elit sodales scelerisque. +dagondesign.com|38|38|38|emiller11@adobe.com|Y|Donec odio justo, sollicitudin ut, suscipit a, feugiat et, eros. Vestibulum ac est lacinia nisi venenatis tristique. +macromedia.com|39|39|39|bjosephov12@youtube.com|Y|Nulla ac enim. In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. +virginia.edu|40|40|40|bashbe13@xinhuanet.com|Y|Proin risus. +wsj.com|41|41|41||Y|Praesent lectus. Vestibulum quam sapien, varius ut, blandit non, interdum in, ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Duis faucibus accumsan odio. +dedecms.com|42|42|42|brain15@sourceforge.net|Y|Sed ante. +ovh.net|43|43|43|gbrehat16@livejournal.com|N|Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. +nyu.edu|44|44|44|fguarnier17@weebly.com|N|Nullam orci pede, venenatis non, sodales sed, tincidunt eu, felis. +kickstarter.com|45|45|45|byes18@google.pl|Y|Pellentesque eget nunc. +about.com|46|46|46|tweine19@wikispaces.com|Y|Integer ac neque. Duis bibendum. +prlog.org|47|47|47|cweeden1a@cocolog-nifty.com|N|Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Donec pharetra, magna vestibulum aliquet ultrices, erat tortor sollicitudin mi, sit amet lobortis sapien sapien non mi. Integer ac neque. +sciencedaily.com|48|48|48|wforcade1b@sciencedaily.com|N|Etiam justo. Etiam pretium iaculis justo. +plala.or.jp|49|49|49|gcarleton1c@quantcast.com|Y|Phasellus sit amet erat. Nulla tempus. +psu.edu|50|50|50|jabbati1d@omniture.com|N|Morbi non quam nec dui luctus rutrum. Nulla tellus. In sagittis dui vel nisl. diff --git a/src/registrar/tests/data/test_organization_adhoc_.txt b/src/registrar/tests/data/test_organization_adhoc_.txt new file mode 100644 index 000000000..ecab84a6a --- /dev/null +++ b/src/registrar/tests/data/test_organization_adhoc_.txt @@ -0,0 +1,51 @@ +orgid|orgname|orgstreet|orgcity|orgstate|orgzip|orgcountrycode +1|Flashdog|298 Monument Hill|Lakeland|Florida|33805|US +2|Gigaclub|782 Mosinee Lane|Alexandria|Louisiana|71307|US +3|Midel|376 Donald Pass|Waco|Texas|76705|US +4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US +5|Kwideo|01 Lotheville Place|Beaumont|Texas|77713|US +6|Brainverse|8 Jenifer Point|Fort Myers|Florida|33994|US +7|Brainsphere|74264 Reinke Place|Flint|Michigan|48550|US +8|Pixoboo|40090 Lillian Avenue|Metairie|Louisiana|70033|US +9|Topicshots|99331 Quincy Alley|Cleveland|Ohio|44177|US +10|Eayo|17364 Vahlen Avenue|El Paso|Texas|88558|US +11|Myworks|961 Kim Park|Honolulu|Hawaii|96845|US +12|Flashset|262 Mcguire Parkway|Rochester|New York|14683|US +13|Quatz|8 Forest Street|Warren|Ohio|44485|US +14|Kazio|928 Carey Plaza|Miami|Florida|33196|US +15|DabZ|05350 Claremont Circle|Lexington|Kentucky|40581|US +16|Livepath|698 5th Crossing|Boca Raton|Florida|33499|US +17|Centimia|82 Packers Court|Simi Valley|California|93094|US +18|Avavee|4 Old Gate Center|Tucson|Arizona|85710|US +19|Wikizz|74785 Oak Valley Crossing|Phoenix|Arizona|85040|US +20|Wikivu|877 Gulseth Park|Tallahassee|Florida|32309|US +21|Brainbox|8 Esker Lane|Lexington|Kentucky|40524|US +22|Jaxworks|2 Prairieview Street|Young America|Minnesota|55573|US +23|Youfeed|191 Ramsey Junction|Suffolk|Virginia|23436|US +24|Ntags|24 Melby Court|Kansas City|Missouri|64136|US +25|Realblab|119 Butternut Avenue|Dallas|Texas|75323|US +26|Trudeo|69 Cordelia Park|Palmdale|California|93591|US +27|Wordware|74540 Jenifer Pass|Lake Charles|Louisiana|70607|US +28|Jaxnation|80974 Homewood Avenue|Philadelphia|Pennsylvania|19160|US +29|Latz|01989 Red Cloud Hill|Columbus|Ohio|43226|US +30|Fivespan|0 Ryan Plaza|Honolulu|Hawaii|96805|US +31|Youfeed|24930 Hoard Park|San Antonio|Texas|78260|US +32|Browsetype|49 Waxwing Circle|Oklahoma City|Oklahoma|73119|US +33|Oba|8426 Thompson Parkway|Anaheim|California|92825|US +34|Yodo|64815 Thackeray Crossing|Salinas|California|93907|US +35|Thoughtstorm|79 Del Sol Drive|Evansville|Indiana|47719|US +36|Yamia|2 Marquette Junction|Newark|New Jersey|07195|US +37|Demimbu|14 American Ash Trail|Bronx|New York|10474|US +38|Rhybox|6234 Cambridge Drive|Fort Lauderdale|Florida|33305|US +39|Fivespan|4 Fair Oaks Terrace|Phoenix|Arizona|85040|US +40|Skipfire|4 Dayton Circle|Mesquite|Texas|75185|US +41|Thoughtmix|52 Claremont Avenue|Indianapolis|Indiana|46207|US +42|Meembee|59 Green Ridge Park|Gainesville|Georgia|30506|US +43|Trudeo|54 Schurz Place|Chicago|Illinois|60609|US +44|Tazz|722 Bunker Hill Place|Bronx|New York|10454|US +45|Skiba|8776 Pennsylvania Way|Fayetteville|North Carolina|28314|US +46|Zoomlounge|56 Sloan Circle|Evansville|Indiana|47712|US +47|Cogilith|7 American Ash Trail|Houston|Texas|77255|US +48|Browsebug|15903 Stephen Hill|Arlington|Virginia|22244|US +49|Yamia|9144 Graedel Crossing|Lehigh Acres|Florida|33972|US +501|Lazzy|4958 Kensington Alley|Fayetteville|North Carolina|28305|US From 83d1db648543600697c001b62c42537199fb0c5f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 09:41:23 -0700 Subject: [PATCH 30/88] Add more test data --- .../tests/data/test_domain_types_adhoc.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/registrar/tests/data/test_domain_types_adhoc.txt diff --git a/src/registrar/tests/data/test_domain_types_adhoc.txt b/src/registrar/tests/data/test_domain_types_adhoc.txt new file mode 100644 index 000000000..d266fa3f3 --- /dev/null +++ b/src/registrar/tests/data/test_domain_types_adhoc.txt @@ -0,0 +1,18 @@ +domaintypeid|domaintype|code|active +14|Federal - Legislative|FL|Y +15|Federal - Judicial|FJ|Y +16|Interstate|IA|Y +1|Commercial|CO|N +2|Courts|CT|N +3|Federal Government Program|FD|N +4|Federal - Executive|FE|Y +5|Foreign|FO|N +6|Individual|IN|N +7|Military|MI|N +8|Not Top-Level|NT|N +9|State|ST|Y +10|Tribal|TN|Y +11|Domain type 1|1|N +12|County|CY|Y +13|City|CI|Y +17|Independent Intrastate|II|Y \ No newline at end of file From 019b139e8d1fcaafb30c6c35939aff7f677910ba Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:17:01 -0700 Subject: [PATCH 31/88] Fix typo --- .../commands/utility/extra_transition_domain_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index d4af2da55..f00f72bc2 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -149,7 +149,7 @@ class LoadExtraTransitionDomain: if not all_transition_domains.exists(): raise ValueError("No TransitionDomain objects exist.") - updated_trasition_domains = [] + updated_transition_domains = [] for transition_domain in all_transition_domains: domain_name = transition_domain.domain_name.upper() updated_transition_domain = transition_domain @@ -181,7 +181,7 @@ class LoadExtraTransitionDomain: f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}" ) - updated_trasition_domains.append(updated_transition_domain) + updated_transition_domains.append(updated_transition_domain) # If we run into an exception on this domain, @@ -197,7 +197,7 @@ class LoadExtraTransitionDomain: logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - updated {len(updated_trasition_domains)} transition domain entries + updated {len(updated_transition_domains)} transition domain entries {TerminalColors.ENDC} """ ) From 2b2a0c0fba0407e95d881f1a51097856bca1957d Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Nov 2023 11:59:37 -0600 Subject: [PATCH 32/88] Updates to prompts and print statements for master migration to be easier to work with --- .../commands/agency_data_extractor.py | 37 +++++++- .../commands/load_transition_domain.py | 4 + .../commands/master_domain_migrations.py | 47 ++++----- .../utility/extra_transition_domain_helper.py | 3 +- .../commands/utility/terminal_helper.py | 95 +++++++++++++++++-- 5 files changed, 149 insertions(+), 37 deletions(-) diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index 3b7931f8d..d5b304a8a 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -78,6 +78,12 @@ class Command(BaseCommand): possibly_unused_agencies.append(agency) TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") + matched_agencies = [] + for agency in provided_agencies: + if agency in existing_agencies: + matched_agencies.append(agency) + TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}") + # Print the summary of findings # 1 - Print the list of agencies in the NEW list, which we do not already have # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? @@ -87,6 +93,9 @@ class Command(BaseCommand): possibly_unused_agencies_as_string = "{}".format( ",\n ".join(map(str, possibly_unused_agencies)) ) + matched_agencies_as_string = "{}".format( + ",\n ".join(map(str, matched_agencies)) + ) logger.info(f""" {TerminalColors.OKGREEN} @@ -96,6 +105,8 @@ class Command(BaseCommand): {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED (These are agencies that are in the given agency file AND in our system already) + {TerminalColors.YELLOW}{matched_agencies_as_string} + {TerminalColors.OKGREEN} {len(new_agencies)} AGENCIES TO ADD: These agencies were in the provided agency file, but are not in our system. @@ -139,13 +150,19 @@ class Command(BaseCommand): new_agencies = self.extract_agencies(agency_data_file, sep, debug) hard_coded_agencies = DomainApplication.AGENCIES + transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct() + print(transition_domain_agencies) + + merged_agencies = new_agencies for agency in hard_coded_agencies: if agency not in merged_agencies: merged_agencies.append(agency) - - transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency').distinct() - print(transition_domain_agencies) + + merged_transition_agencies = new_agencies + for agency in transition_domain_agencies: + if agency not in merged_transition_agencies: + merged_transition_agencies.append(agency) prompt_successful = False @@ -208,4 +225,16 @@ class Command(BaseCommand): f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." f"\n\n{len(merged_agencies)} TOTAL\n\n" ) - self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") \ No newline at end of file + self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== MERGED LISTS (transition domain + agency file) ============================" + f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." + f"\n\n{len(merged_agencies)} TOTAL\n\n" + ) + self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List") \ No newline at end of file diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 31ccf1cd6..cb85bd2d8 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -381,8 +381,10 @@ class Command(BaseCommand): # Start parsing the main file and create TransitionDomain objects logger.info("Reading domain-contacts data file %s", domain_contacts_filename) + total_lines = TerminalHelper.get_file_line_count(domain_contacts_filename) with open(domain_contacts_filename, "r") as domain_contacts_file: for row in csv.reader(domain_contacts_file, delimiter=sep): + # TerminalHelper.printProgressBar(total_rows_parsed, total_lines) total_rows_parsed += 1 # fields are just domain, userid, role @@ -394,6 +396,8 @@ class Command(BaseCommand): new_entry_email = "" new_entry_emailSent = False # set to False by default + TerminalHelper.print_conditional(debug_on, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") + # PART 1: Get the status if new_entry_domain_name not in domain_status_dictionary: # This domain has no status...default to "Create" diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index faba8a037..ea0e827fa 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -296,25 +296,27 @@ class Command(BaseCommand): command_string += f"--limitParse {debug_max_entries_to_parse} " # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running load_transition_domain script", ) # TODO: make this somehow run inside TerminalHelper prompt - call_command( - command_script, - f"{file_location+domain_contacts_filename}", - f"{file_location+contacts_filename}", - f"{file_location+domain_statuses_filename}", - sep=sep, - resetTable=reset_table, - debug=debug_on, - limitParse=debug_max_entries_to_parse, - ) + if proceed: + call_command( + command_script, + f"{file_location+domain_contacts_filename}", + f"{file_location+contacts_filename}", + f"{file_location+domain_statuses_filename}", + sep=sep, + resetTable=reset_table, + debug=debug_on, + limitParse=debug_max_entries_to_parse, + directory=file_location + ) def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" @@ -324,16 +326,16 @@ class Command(BaseCommand): if debug_on: command_string += "--debug " # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running transfer_transition_domains_to_domains script", ) - # TODO: make this somehow run inside TerminalHelper prompt - call_command(command_script) + if proceed: + call_command(command_script) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" @@ -341,16 +343,17 @@ class Command(BaseCommand): command_script = "send_domain_invitations" command_string = f"./manage.py {command_script} -s" # Execute the command string + proceed = False if prompts_enabled: - system_exit_on_terminate = True - TerminalHelper.prompt_for_execution( - system_exit_on_terminate, + proceed = TerminalHelper.prompt_for_execution( + False, command_string, "Running send_domain_invitations script", ) # TODO: make this somehow run inside TerminalHelper prompt - call_command(command_script, send_emails=True) + if proceed: + call_command(command_script, send_emails=True) def run_migration_scripts( self, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index f00f72bc2..734beb408 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -197,7 +197,8 @@ class LoadExtraTransitionDomain: logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - updated {len(updated_transition_domains)} transition domain entries + updated {len(updated_trasition_domains)} transition domain entries: + {TerminalHelper.array_as_string(updated_trasition_domains)} {TerminalColors.ENDC} """ ) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 78ffb525f..5ec9408cd 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -70,12 +70,45 @@ class TerminalHelper: else: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + @staticmethod + def query_yes_no_exit(question: str, default="yes") -> bool: + """Ask a yes/no question via raw_input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + """ + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit", "s": "skip"} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + logger.info(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return valid[default] + elif choice in valid: + if valid[choice] == "exit": + sys.exit() + return valid[choice] + else: + logger.info("Please respond with a valid selection.\n") + # @staticmethod - # def array_as_string(array_to_convert: []) -> str: - # array_as_string = "{}".format( - # ", ".join(map(str, array_to_convert)) - # ) - # return array_as_string + def array_as_string(array_to_convert: []) -> str: + array_as_string = "{}".format( + "\n".join(map(str, array_to_convert)) + ) + return array_as_string @staticmethod def print_conditional( @@ -118,13 +151,13 @@ class TerminalHelper: Returns true if the user responds (y), Returns false if the user responds (n)""" - action_description_for_selecting_no = "skip" + action_description_for_selecting_no = "skip, E = exit" if system_exit_on_terminate: - action_description_for_selecting_no = "exit" + action_description_for_selecting_no = "exit, S = skip" # Allow the user to inspect the command string # and ask if they wish to proceed - proceed_execution = TerminalHelper.query_yes_no( + proceed_execution = TerminalHelper.query_yes_no_exit( f"""{TerminalColors.OKCYAN} ===================================================== {prompt_title} @@ -139,10 +172,52 @@ class TerminalHelper: # If the user decided to proceed return true. # Otherwise, either return false or exit this subroutine. - if not proceed_execution: + if proceed_execution == False: if system_exit_on_terminate: sys.exit() return False - + if proceed_execution == "skip": + return False return True + @staticmethod + def get_file_line_count(filepath: str) -> int: + with open(filepath,'r') as file: + li = file.readlines() + total_line = len(li) + return total_line + + + + @staticmethod + def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"): + """ + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : positive number of decimals in percent complete (Int) + length - Optional : character length of bar (Int) + fill - Optional : bar fill character (Str) + printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) + """ + + """ + # Initial call to print 0% progress + printProgressBar(0, l, prefix = 'Progress:', suffix = 'Complete', length = 50) + for i, item in enumerate(items): + # Do stuff... + time.sleep(0.1) + # Update Progress Bar + printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) + """ + + percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) + filledLength = int(length * iteration // total) + bar = fill * filledLength + '-' * (length - filledLength) + print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) + # Print New Line on Complete + if iteration == total: + print() \ No newline at end of file From 4595c2b5a922a0940d6d382ae48c87511354490d Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Nov 2023 12:00:01 -0600 Subject: [PATCH 33/88] Extra info for TransitionDomain objects __str__ function --- src/registrar/models/transition_domain.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index a719defe1..a31b27e7e 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -80,8 +80,15 @@ class TransitionDomain(TimeStampedModel): def __str__(self): return ( - f"username: {self.username} " - f"domainName: {self.domain_name} " - f"status: {self.status} " - f"email sent: {self.email_sent} " + f"\n-----TRANSITION DOMAIN------\n" + f"domainName: {self.domain_name}, \n" + f"username: {self.username}, \n" + f"status: {self.status}, \n" + f"email sent: {self.email_sent}, \n" + f"organization type: {self.organization_type}, \n" + f"organization_name: {self.organization_name}, \n" + f"federal_type: {self.federal_type}, \n" + f"federal_agency: {self.federal_agency}, \n" + f"epp_creation_date: {self.epp_creation_date}, \n" + f"epp_expiration_date: {self.epp_expiration_date}, \n" ) From 661b765f0e9b506df205c5c8b3009fec3199614e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:12:01 -0700 Subject: [PATCH 34/88] Basic test cases --- .../tests/data/test_escrow_domains_daily.txt | 0 ...adhoc_.txt => test_organization_adhoc.txt} | 0 .../test_transition_domain_migrations.py | 51 ++++++++++++------- 3 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 src/registrar/tests/data/test_escrow_domains_daily.txt rename src/registrar/tests/data/{test_organization_adhoc_.txt => test_organization_adhoc.txt} (100%) diff --git a/src/registrar/tests/data/test_escrow_domains_daily.txt b/src/registrar/tests/data/test_escrow_domains_daily.txt new file mode 100644 index 000000000..e69de29bb diff --git a/src/registrar/tests/data/test_organization_adhoc_.txt b/src/registrar/tests/data/test_organization_adhoc.txt similarity index 100% rename from src/registrar/tests/data/test_organization_adhoc_.txt rename to src/registrar/tests/data/test_organization_adhoc.txt diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 11324cb37..811a6013d 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -1,3 +1,4 @@ +from io import StringIO from django.test import TestCase from registrar.models import ( @@ -10,9 +11,9 @@ from registrar.models import ( ) from django.core.management import call_command +from unittest.mock import patch - -class TestLogins(TestCase): +class TestMigrations(TestCase): """ """ @@ -27,37 +28,49 @@ class TestLogins(TestCase): self.test_contact_filename = "test_contacts.txt" self.test_domain_status_filename = "test_domain_statuses.txt" + # Files for parsing additional TransitionDomain data + self.test_agency_adhoc_filename = "test_agency_adhoc.txt" + self.test_authority_adhoc_filename = "test_authority_adhoc.txt" + self.test_domain_additional = "test_domain_additional.txt" + self.test_domain_types_adhoc = "test_domain_types_adhoc.txt" + self.test_escrow_domains_daily = "test_escrow_domains_daily" + self.test_organization_adhoc = "test_organization_adhoc.txt" + def tearDown(self): - super().tearDown() + # Delete domain information TransitionDomain.objects.all().delete() Domain.objects.all().delete() DomainInvitation.objects.all().delete() DomainInformation.objects.all().delete() + + # Delete users User.objects.all().delete() UserDomainRole.objects.all().delete() def run_load_domains(self): - call_command( - "load_transition_domain", - f"{self.test_data_file_location}/{self.test_domain_contact_filename}", - f"{self.test_data_file_location}/{self.test_contact_filename}", - f"{self.test_data_file_location}/{self.test_domain_status_filename}", - ) + with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no', return_value=True): + call_command( + "load_transition_domain", + f"{self.test_data_file_location}/{self.test_domain_contact_filename}", + f"{self.test_data_file_location}/{self.test_contact_filename}", + f"{self.test_data_file_location}/{self.test_domain_status_filename}", + ) def run_transfer_domains(self): call_command("transfer_transition_domains_to_domains") def run_master_script(self): - call_command( - "master_domain_migrations", - runMigrations=True, - migrationDirectory=f"{self.test_data_file_location}", - migrationFilenames=( - f"{self.test_domain_contact_filename}," - f"{self.test_contact_filename}," - f"{self.test_domain_status_filename}" - ), - ) + with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no', return_value=True): + call_command( + "master_domain_migrations", + runMigrations=True, + migrationDirectory=f"{self.test_data_file_location}", + migrationFilenames=( + f"{self.test_domain_contact_filename}," + f"{self.test_contact_filename}," + f"{self.test_domain_status_filename}" + ), + ) def compare_tables( self, From 642c23d3b7f2bfe14e341eeabf4731bb6b525917 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:08:02 -0700 Subject: [PATCH 35/88] Fix bulk add bug --- .../transfer_transition_domains_to_domains.py | 14 ++++++- .../utility/extra_transition_domain_helper.py | 37 +++++++++++++------ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index d3f8d1aa9..7ed0ba009 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -640,8 +640,20 @@ class Command(BaseCommand): {TerminalColors.ENDC}""" ) + # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) - DomainInvitation.objects.bulk_create(domain_invitations_to_create) + #DomainInvitation.objects.bulk_create(domain_invitations_to_create) + + # Then, create DomainInvitation objects + for invitation in domain_invitations_to_create: + existing_domain = Domain.objects.filter(name=invitation.domain.name) + # Make sure the related Domain object is saved + if existing_domain.exists(): + invitation.domain = existing_domain.get() + else: + # Raise an err for now + raise Exception(f"Domain {existing_domain} wants to be added but doesn't exist in the DB") + invitation.save() # ====================================================== # ================= DOMAIN INFORMATION ================= diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index f00f72bc2..3e4b3e914 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -150,6 +150,7 @@ class LoadExtraTransitionDomain: raise ValueError("No TransitionDomain objects exist.") updated_transition_domains = [] + failed_transition_domains = [] for transition_domain in all_transition_domains: domain_name = transition_domain.domain_name.upper() updated_transition_domain = transition_domain @@ -174,33 +175,47 @@ class LoadExtraTransitionDomain: domain_name, transition_domain ) + # Check if the instance has changed before saving + #if updated_transition_domain.__dict__ != transition_domain.__dict__: updated_transition_domain.save() + updated_transition_domains.append(updated_transition_domain) + self.parse_logs.display_logs_by_domain_name(domain_name) logger.info( f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}" ) - updated_transition_domains.append(updated_transition_domain) - # If we run into an exception on this domain, # Just skip over it and log that it happened. except Exception as err: logger.debug(err) - logger.info( + logger.error( f"{TerminalColors.FAIL}" f"Exception encountered on {domain_name}. Could not update." f"{TerminalColors.ENDC}" ) - raise err - logger.info( - f"""{TerminalColors.OKGREEN} - ============= FINISHED =============== - updated {len(updated_transition_domains)} transition domain entries - {TerminalColors.ENDC} - """ - ) + failed_transition_domains.append(domain_name) + + failed_count = len(failed_transition_domains) + if failed_count == 0: + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED =============== + Updated {len(updated_transition_domains)} transition domain entries + {TerminalColors.ENDC} + """ + ) + else: + logger.error( + f"""{TerminalColors.FAIL} + ============= FINISHED WITH ERRORS =============== + Updated {len(updated_transition_domains)} transition domain entries, + Failed to update {failed_count} transition domain entries + {TerminalColors.ENDC} + """ + ) def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it From 8d044618a95b57f7ab5fdf95329786f6d2ff756a Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:31:14 -0700 Subject: [PATCH 36/88] Fixed broken test cases --- .../test_transition_domain_migrations.py | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 811a6013d..fc1bf39cb 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -139,20 +139,19 @@ class TestMigrations(TestCase): total_domain_invitations = {len(DomainInvitation.objects.all())} """ ) - - self.assertTrue(total_missing_domains == expected_missing_domains) - self.assertTrue(total_duplicate_domains == expected_duplicate_domains) - self.assertTrue( - total_missing_domain_informations == expected_missing_domain_informations + self.assertEqual(total_missing_domains, expected_missing_domains) + self.assertEqual(total_duplicate_domains, expected_duplicate_domains) + self.assertEqual( + total_missing_domain_informations, expected_missing_domain_informations ) - self.assertTrue( - total_missing_domain_invitations == expected_missing_domain_invitations + self.assertEqual( + total_missing_domain_invitations, expected_missing_domain_invitations ) - self.assertTrue(total_transition_domains == expected_total_transition_domains) - self.assertTrue(total_domains == expected_total_domains) - self.assertTrue(total_domain_informations == expected_total_domain_informations) - self.assertTrue(total_domain_invitations == expected_total_domain_invitations) + self.assertEqual(total_transition_domains, expected_total_transition_domains) + self.assertEqual(total_domains, expected_total_domains) + self.assertEqual(total_domain_informations, expected_total_domain_informations) + self.assertEqual(total_domain_invitations, expected_total_domain_invitations) def test_master_migration_functions(self): """Run the full master migration script using local test data. @@ -167,14 +166,12 @@ class TestMigrations(TestCase): # migration script does, but add assert statements) expected_total_transition_domains = 8 expected_total_domains = 4 - expected_total_domain_informations = 0 + expected_total_domain_informations = 4 expected_total_domain_invitations = 7 expected_missing_domains = 0 expected_duplicate_domains = 0 - # we expect 8 missing domain invites since the - # migration does not auto-login new users - expected_missing_domain_informations = 8 + expected_missing_domain_informations = 0 # we expect 1 missing invite from anomaly.gov (an injected error) expected_missing_domain_invitations = 1 self.compare_tables( @@ -221,12 +218,12 @@ class TestMigrations(TestCase): # Analyze the tables expected_total_transition_domains = 8 expected_total_domains = 4 - expected_total_domain_informations = 0 + expected_total_domain_informations = 4 expected_total_domain_invitations = 7 expected_missing_domains = 0 expected_duplicate_domains = 0 - expected_missing_domain_informations = 8 + expected_missing_domain_informations = 0 expected_missing_domain_invitations = 1 self.compare_tables( expected_total_transition_domains, @@ -255,12 +252,12 @@ class TestMigrations(TestCase): # Analyze the tables expected_total_transition_domains = 8 expected_total_domains = 4 - expected_total_domain_informations = 3 + expected_total_domain_informations = 4 expected_total_domain_invitations = 7 expected_missing_domains = 0 expected_duplicate_domains = 0 - expected_missing_domain_informations = 1 + expected_missing_domain_informations = 0 expected_missing_domain_invitations = 1 self.compare_tables( expected_total_transition_domains, From 9fc7ec4bea8babed9bd824b407d0968d1889c9d1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 14:06:01 -0700 Subject: [PATCH 37/88] Update test_transition_domain_migrations.py --- src/registrar/tests/test_transition_domain_migrations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index fc1bf39cb..fbe400692 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -48,7 +48,7 @@ class TestMigrations(TestCase): UserDomainRole.objects.all().delete() def run_load_domains(self): - with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no', return_value=True): + with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): call_command( "load_transition_domain", f"{self.test_data_file_location}/{self.test_domain_contact_filename}", @@ -60,7 +60,7 @@ class TestMigrations(TestCase): call_command("transfer_transition_domains_to_domains") def run_master_script(self): - with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no', return_value=True): + with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): call_command( "master_domain_migrations", runMigrations=True, @@ -131,7 +131,7 @@ class TestMigrations(TestCase): total_missing_domains = {len(missing_domains)} total_duplicate_domains = {len(duplicate_domains)} total_missing_domain_informations = {len(missing_domain_informations)} - total_missing_domain_invitations = {len(missing_domain_invites)} + total_missing_domain_invitations = {total_missing_domain_invitations} total_transition_domains = {len(TransitionDomain.objects.all())} total_domains = {len(Domain.objects.all())} From a4fcca23bafac0b35f9c5188bb05e3041bfb02b9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:54:34 -0700 Subject: [PATCH 38/88] Minor rewrite / expand tests --- .../commands/load_transition_domain.py | 13 ++++- .../commands/utility/epp_data_containers.py | 25 +++++++-- .../utility/extra_transition_domain_helper.py | 47 +++++++--------- .../utility/transition_domain_arguments.py | 51 +++++++++--------- .../tests/data/test_agency_adhoc.txt | 48 +---------------- .../tests/data/test_authority_adhoc.txt | 54 ++----------------- .../tests/data/test_domain_additional.txt | 54 ++----------------- .../tests/data/test_domain_contacts.txt | 2 +- .../tests/data/test_domain_statuses.txt | 2 +- .../tests/data/test_organization_adhoc.txt | 48 +---------------- .../test_transition_domain_migrations.py | 13 ++++- 11 files changed, 98 insertions(+), 259 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index cb85bd2d8..1b3cf8642 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -71,6 +71,9 @@ class Command(BaseCommand): parser.add_argument( "--directory", default="migrationdata", help="Desired directory" ) + parser.add_argument( + "--infer_filenames", default=False, help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." + ) parser.add_argument( "--agency_adhoc_filename", default=EnumFilenames.AGENCY_ADHOC.value[1], @@ -97,6 +100,11 @@ class Command(BaseCommand): default=EnumFilenames.ORGANIZATION_ADHOC.value[1], help="Defines the filename for domain type adhocs", ) + parser.add_argument( + "--authority_adhoc_filename", + default=EnumFilenames.AUTHORITY_ADHOC.value[1], + help="Defines the filename for domain type adhocs", + ) def print_debug_mode_statements( self, debug_on: bool, debug_max_entries_to_parse: int @@ -551,6 +559,9 @@ class Command(BaseCommand): system_exit_on_terminate=False, info_to_inspect=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING + ==Master data file== + domain_additional_filename: {domain_additional_filename} + ==Federal agency information== agency_adhoc_filename: {agency_adhoc_filename} @@ -563,7 +574,7 @@ class Command(BaseCommand): ==Creation date / expiration date information== domain_escrow_filename: {domain_escrow_filename} - domain_additional_filename: {domain_additional_filename} + ==Containing directory== directory: {directory} """, prompt_title=title, diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 6dabb78a2..36b5e3f17 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -89,12 +89,27 @@ class EnumFilenames(Enum): # We are sourcing data from many different locations, so its better to track this # as an Enum rather than multiple spread out variables. # We store the "type" as [0], and we store the "default_filepath" as [1]. - AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") + AGENCY_ADHOC = ( + "agency_adhoc", + "agency.adhoc.dotgov.txt" + ) DOMAIN_ADDITIONAL = ( "domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt", ) - DOMAIN_ESCROW = ("domain_escrow", "escrow_domains.daily.dotgov.GOV.txt") - DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") - ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") - AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt") + DOMAIN_ESCROW = ( + "domain_escrow", + "escrow_domains.daily.dotgov.GOV.txt" + ) + DOMAIN_ADHOC = ( + "domain_adhoc", + "domaintypes.adhoc.dotgov.txt" + ) + ORGANIZATION_ADHOC = ( + "organization_adhoc", + "organization.adhoc.dotgov.txt" + ) + AUTHORITY_ADHOC = ( + "authority_adhoc", + "authority.adhoc.dotgov.txt" + ) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index af392b940..e98954c60 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -136,11 +136,10 @@ class LoadExtraTransitionDomain: def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() - arguments = options.args_extra_transition_domain() # Reads and parses migration files self.parsed_data_container = ExtraTransitionDomain(**arguments) - self.parsed_data_container.parse_all_files() + self.parsed_data_container.parse_all_files(options.infer_filenames) def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content @@ -683,24 +682,14 @@ class ExtraTransitionDomain: strip_date_regex = re.compile(r"(?:.*\/)?(\d+)\.(.+)") - def __init__( - self, - agency_adhoc_filename=EnumFilenames.AGENCY_ADHOC.value[1], - domain_additional_filename=EnumFilenames.DOMAIN_ADDITIONAL.value[1], - domain_escrow_filename=EnumFilenames.DOMAIN_ESCROW.value[1], - domain_adhoc_filename=EnumFilenames.DOMAIN_ADHOC.value[1], - organization_adhoc_filename=EnumFilenames.ORGANIZATION_ADHOC.value[1], - authority_adhoc_filename=EnumFilenames.AUTHORITY_ADHOC.value[1], - directory="migrationdata", - sep="|", - ): + def __init__(self, options: TransitionDomainArguments): # Add a slash if the last character isn't one - if directory and directory[-1] != "/": - directory += "/" - self.directory = directory - self.seperator = sep + if options.directory and options.directory[-1] != "/": + options.directory += "/" + self.directory = options.directory + self.seperator = options.sep - self.all_files = glob.glob(f"{directory}*") + self.all_files = glob.glob(f"{self.directory}*") # Create a set with filenames as keys for quick lookup self.all_files_set = {os.path.basename(file) for file in self.all_files} @@ -713,37 +702,37 @@ class ExtraTransitionDomain: pattern_map_params = [ ( EnumFilenames.AGENCY_ADHOC, - agency_adhoc_filename, + options.agency_adhoc_filename, AgencyAdhoc, "agencyid", ), ( EnumFilenames.DOMAIN_ADDITIONAL, - domain_additional_filename, + options.domain_additional_filename, DomainAdditionalData, "domainname", ), ( EnumFilenames.DOMAIN_ESCROW, - domain_escrow_filename, + options.domain_escrow_filename, DomainEscrow, "domainname", ), ( EnumFilenames.DOMAIN_ADHOC, - domain_adhoc_filename, + options.domain_adhoc_filename, DomainTypeAdhoc, "domaintypeid", ), ( EnumFilenames.ORGANIZATION_ADHOC, - organization_adhoc_filename, + options.organization_adhoc_filename, OrganizationAdhoc, "orgid", ), ( EnumFilenames.AUTHORITY_ADHOC, - authority_adhoc_filename, + options.authority_adhoc_filename, AuthorityAdhoc, "authorityid", ), @@ -758,7 +747,7 @@ class ExtraTransitionDomain: pattern_map_params must adhere to this format: [ - (field_type, filename, data_type, id_field), + (file_type, filename, data_type, id_field), ] vars: @@ -800,8 +789,8 @@ class ExtraTransitionDomain: def parse_all_files(self, infer_filenames=True): """Clears all preexisting data then parses each related CSV file. - overwrite_existing_data: bool -> Determines if we should clear - file_data.data if it already exists + infer_filenames: bool -> Determines if we should try to + infer the filename if a default is passed in """ self.clear_file_data() for name, value in self.file_data.items(): @@ -822,13 +811,13 @@ class ExtraTransitionDomain: continue # Infer filename logic # - # This mode is used for development and testing only. Rather than having + # This mode is used for internal development use and testing only. Rather than having # to manually define the filename each time, we can infer what the filename # actually is. # Not intended for use outside of that, as it is better to assume # the end-user wants to be specific. - logger.warning("Attempting to infer filename" f" for file: {filename}.") + logger.warning(f"Attempting to infer filename: {filename}") for filename in self.all_files: default_name = name.value[1] match = value.try_infer_filename(filename, default_name) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 459eab6d3..e95425255 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -1,33 +1,30 @@ +from dataclasses import dataclass, field +from typing import Optional + +from registrar.management.commands.utility.epp_data_containers import EnumFilenames + +@dataclass class TransitionDomainArguments: """Stores arguments for load_transition_domain""" - def __init__(self, **options): - # Settings # - self.directory = options.get("directory") - self.sep = options.get("sep") - self.limitParse = options.get("limitParse") + # Settings # + directory: Optional[str] = field(default="migrationdata", repr=True) + sep: Optional[str] = field(default="|", repr=True) + limitParse: Optional[int] = field(default=None, repr=True) - # Filenames # - ## Adhocs ## - self.agency_adhoc_filename = options.get("agency_adhoc_filename") - self.domain_adhoc_filename = options.get("domain_adhoc_filename") - self.organization_adhoc_filename = options.get("organization_adhoc_filename") + # Filenames # + ## Adhocs ## + agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True) + domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True) + organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True) + authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) - ## Data files ## - self.domain_additional_filename = options.get("domain_additional_filename") - self.domain_contacts_filename = options.get("domain_contacts_filename") - self.domain_statuses_filename = options.get("domain_statuses_filename") + ## Data files ## + domain_additional_filename: Optional[str] = field(default=None, repr=True) + domain_contacts_filename: Optional[str] = field(default=None, repr=True) + domain_statuses_filename: Optional[str] = field(default=None, repr=True) - # Flags # - self.debug = options.get("debug") - self.resetTable = options.get("resetTable") - - def args_extra_transition_domain(self): - return { - "agency_adhoc_filename": self.agency_adhoc_filename, - "domain_adhoc_filename": self.domain_adhoc_filename, - "organization_adhoc_filename": self.organization_adhoc_filename, - "domain_additional_filename": self.domain_additional_filename, - "directory": self.directory, - "sep": self.sep, - } + # Flags # + debug: Optional[bool] = field(default=False, repr=True) + resetTable: Optional[bool] = field(default=False, repr=True) + infer_filenames: Optional[bool] = field(default=False, repr=True) diff --git a/src/registrar/tests/data/test_agency_adhoc.txt b/src/registrar/tests/data/test_agency_adhoc.txt index 1b0108aee..3de43d2d9 100644 --- a/src/registrar/tests/data/test_agency_adhoc.txt +++ b/src/registrar/tests/data/test_agency_adhoc.txt @@ -2,50 +2,4 @@ agencyid|agencyname|active|isfederal 1|Thoughtstorm|N|Y 2|Minyx|Y|N 3|Demivee|N|Y -4|InnoZ|Y|Y -5|Jayo|N|Y -6|Avaveo|N|Y -7|Thoughtmix|N|N -8|Photofeed|Y|N -9|Gabtype|N|Y -10|Youfeed|Y|N -11|Kwimbee|N|Y -12|Linkbridge|N|Y -13|Wikibox|Y|Y -14|Browsezoom|Y|Y -15|Zoozzy|Y|N -16|Mydeo|Y|Y -17|Chatterpoint|N|Y -18|Roodel|N|Y -19|Mybuzz|N|N -20|Thoughtmix|Y|Y -21|Brainlounge|N|Y -22|Quinu|Y|Y -23|Gigaclub|Y|N -24|Eare|Y|N -25|Einti|Y|N -26|Einti|Y|Y -27|Skidoo|Y|Y -28|Aibox|N|Y -29|Zoozzy|Y|Y -30|Centimia|Y|N -31|Einti|N|Y -32|Devcast|Y|N -33|Tagfeed|N|Y -34|Jabbersphere|Y|Y -35|Yamia|Y|Y -36|Fliptune|Y|N -37|Oloo|N|Y -38|Yozio|N|N -39|Brainsphere|Y|Y -40|Geba|Y|Y -41|Wikibox|N|Y -42|Topdrive|Y|Y -43|Lazz|N|N -44|Rooxo|Y|Y -45|Jetwire|N|N -46|Zoomzone|Y|Y -47|Thoughtbridge|Y|N -48|Pixope|Y|N -49|Quatz|N|N -50|Eare|N|Y +4|InnoZ|Y|Y \ No newline at end of file diff --git a/src/registrar/tests/data/test_authority_adhoc.txt b/src/registrar/tests/data/test_authority_adhoc.txt index 007930ccc..aa24e9385 100644 --- a/src/registrar/tests/data/test_authority_adhoc.txt +++ b/src/registrar/tests/data/test_authority_adhoc.txt @@ -1,51 +1,5 @@ authorityid|firstname|middlename|lastname|email|phonenumber|agencyid|addlinfo -1|Gregoor||Kalinke|gkalinke0@indiegogo.com|(773) 1725515|1|Asparagus - Mexican -2|Fayre||Filippozzi|ffilippozzi1@hugedomains.com|(357) 4874280|2|Steampan - Foil -3|Gabey||Lightbody|glightbody2@fc2.com|(332) 8165691|3|Soup - Campbells, Minestrone -4|Seline||Tower|stower3@answers.com|(151) 5396028|4|Kiwi Gold Zespri -5|Rahel||Bruhnsen|rbruhnsen4@google.de|(221) 9271443|5|Miso - Soy Bean Paste -6|Barny||Hopfer|bhopfer5@geocities.jp|(785) 6558321|6|Rice - Jasmine Sented -7|Egan|Loris, slender|Tanslie|etanslie6@auda.org.au|(867) 8852523|7|Compound - Raspberry -8|Meg|Lesser mouse lemur|McLucky|mmclucky7@sciencedirect.com|(207) 4569199|8|Potatoes - Yukon Gold 5 Oz -9|Roarke|American alligator|Thackwray|rthackwray8@un.org|(227) 1557126|9|Schnappes - Peach, Walkers -10|Matteo|Tern, royal|Pancost|mpancost9@typepad.com|(425) 7967200|10|Tortillas - Flour, 12 -11|Wilhelmine||Hradsky|whradskya@tmall.com|(229) 6901308|11|Pail For Lid 1537 -12|Merrile||Dalyiel|mdalyielb@pagesperso-orange.fr|(370) 8234182|12|Table Cloth 90x90 Colour -13|Addy||Jimenez|ajimenezc@angelfire.com|(533) 1368420|13|Wanton Wrap -14|Florella||Tellwright|ftellwrightd@house.gov|(813) 6121895|14|Water - Tonic -15|Jacenta||Flewitt|jflewitte@goo.ne.jp|(884) 4307761|15|Veal - Insides, Grains -16|Nady|Baboon, gelada|Senten|nsentenf@yandex.ru|(996) 6939643|16|Soho Lychee Liqueur -17|Mano|Common palm civet|D'Cruze|mdcruzeg@ox.ac.uk|(114) 9154228|17|Goat - Whole Cut -18|Tadio||Walsh|twalshh@wunderground.com|(490) 6376756|18|Pomello -19|Carey||Boler|cboleri@google.co.jp|(439) 9984218|19|Wasabi Powder -20|Aldus||Denington|adeningtonj@npr.org|(443) 5882262|20|Devonshire Cream -21|Herculie|Horned lark|Delooze|hdeloozek@jimdo.com|(842) 7054442|21|Higashimaru Usukuchi Soy -22|Gertrud||Rosenzwig|grosenzwigl@bloglovin.com|(878) 8890041|22|Apples - Sliced / Wedge -23|Cece||Grimme|cgrimmem@senate.gov|(802) 2135321|23|Capers - Ox Eye Daisy -24|Leann|Bird, red-billed tropic|McGrey|lmcgreyn@cbsnews.com|(681) 8510458|24|Liqueur - Melon -25|Rosabelle|Turtle, eastern box|Rennels|rrennelso@vinaora.com|(447) 9158723|25|Turnip - White, Organic -26|Clay|Duck, white-faced whistling|Acland|caclandp@oracle.com|(387) 6213827|26|Veal - Tenderloin, Untrimmed -27|Devland||Hugland|dhuglandq@ning.com|(391) 8516099|27|Veal - Eye Of Round -28|Cris|Bleu, red-cheeked cordon|Morville|cmorviller@archive.org|(751) 2295767|28|Pie Shells 10 -29|Jehu||Probetts|jprobettss@mac.com|(719) 3208086|29|Pasta - Ravioli -30|Kamilah||Bartalin|kbartalint@devhub.com|(889) 5426094|30|Oranges -31|Katherine|Crane, brolga|Degoe|kdegoeu@weebly.com|(815) 7408114|31|Oranges - Navel, 72 -32|Kassey||Riba|kribav@soup.io|(267) 4032421|32|Sauce - Vodka Blush -33|Marcello||Woodman|mwoodmanw@dell.com|(869) 5497448|33|Potatoes - Instant, Mashed -34|Marie-jeanne||Yo|myox@nymag.com|(414) 2722319|34|Cafe Royale -35|Zerk||Morland|zmorlandy@xing.com|(684) 4155779|35|Muskox - French Rack -36|Rene|Darwin ground finch (unidentified)|Booker|rbookerz@tmall.com|(727) 7131800|36|Onions - Red -37|Romain||Kinnie|rkinnie10@cdc.gov|(967) 3799924|37|Lettuce - Spring Mix -38|Fredra||Denisot|fdenisot11@google.es|(986) 9408987|38|Vinegar - Raspberry -39|Ania||Djurdjevic|adjurdjevic12@wikispaces.com|(854) 8149676|39|Crawfish -40|Gretal|Red-cheeked cordon bleu|Winson|gwinson13@istockphoto.com|(407) 3343406|40|Chips - Miss Vickies -41|Ibby|Squirrel, antelope ground|Bediss|ibediss14@webs.com|(517) 5564511|41|Sobe - Orange Carrot -42|Kingsley||Sawl|ksawl15@reuters.com|(994) 8049936|42|Water - Mineral, Carbonated -43|Syd|Lapwing, southern|Valente|svalente16@whitehouse.gov|(442) 8663735|43|Pork - Sausage, Medium -44|Jsandye||Maylin|jmaylin17@archive.org|(560) 9571021|44|Muffin Hinge Container 6 -45|Beilul||Sedworth|bsedworth18@noaa.gov|(590) 1848805|45|Coriander - Seed -46|Dudley||Note|dnote19@unblog.fr|(674) 5901607|46|Apple - Custard -47|Berte||Forsdicke|bforsdicke1a@theguardian.com|(680) 4006701|47|Paste - Black Olive -48|Gwendolen|Magpie, australian|Drawmer|gdrawmer1b@nba.com|(414) 1746171|48|Extract - Almond -49|Ade||Wilkes|awilkes1c@google.es|(340) 1804264|49|Onions - Pearl -50|Nils||Burnard|nburnard1d@tuttocitta.it|(729) 4332944|50|Basil - Pesto Sauce +1|Gregoor|middle|Kalinke|gkalinke0@indiegogo.com|(773) 172-5515|1|Asparagus - Mexican +2|Fayre||Filippozzi|ffilippozzi1@hugedomains.com|(357) 487-4280|2|Steampan - Foil +3|Gabey||Lightbody|glightbody2@fc2.com|(332) 816-5691|3|Soup - Campbells, Minestrone +4|Seline||Tower|stower3@answers.com|(151) 539-6028|4|Kiwi Gold Zespri \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_additional.txt b/src/registrar/tests/data/test_domain_additional.txt index 15c90326e..b8194daa8 100644 --- a/src/registrar/tests/data/test_domain_additional.txt +++ b/src/registrar/tests/data/test_domain_additional.txt @@ -1,51 +1,5 @@ domainname|domaintypeid|authorityid|orgid|securitycontactemail|dnsseckeymonitor|domainpurpose -indiegogo.com|1|1|1|ggennrich0@utexas.edu|N|Praesent id massa id nisl venenatis lacinia. -gravatar.com|2|2|2|lrome1@uol.com.br|Y|In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. -multiply.com|3|3|3|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. -free.fr|4|4|4|plarderot3@t.co|Y|Morbi quis tortor id nulla ultrices aliquet. Maecenas leo odio, condimentum id, luctus nec, molestie sed, justo. Pellentesque viverra pede ac diam. -washingtonpost.com|5|5|5|dchaim4@yahoo.co.jp|N|Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. Quisque porta volutpat erat. -simplemachines.org|6|6|6|iyaxley5@slashdot.org|N|Phasellus in felis. -ted.com|7|7|7|ktresler6@netscape.com|N|Integer pede justo, lacinia eget, tincidunt eget, tempus vel, pede. Morbi porttitor lorem id ligula. Suspendisse ornare consequat lectus. -diigo.com|8|8|8|dfiggures7@opera.com|Y| -cafepress.com|9|9|9|npowys8@psu.edu|N|Integer a nibh. In quis justo. -slashdot.org|10|10|10||N|Curabitur at ipsum ac tellus semper interdum. Mauris ullamcorper purus sit amet nulla. Quisque arcu libero, rutrum ac, lobortis vel, dapibus at, diam. -shop-pro.jp|11|11|11|laspina@e-recht24.de|N|Aliquam non mauris. Morbi non lectus. -nature.com|12|12|12|wlanonb@bloomberg.com|Y|Praesent id massa id nisl venenatis lacinia. -comsenz.com|13|13|13|jhalloranc@wiley.com|Y|Morbi ut odio. -ed.gov|14|14|14|lpatemand@mit.edu|N|Donec odio justo, sollicitudin ut, suscipit a, feugiat et, eros. Vestibulum ac est lacinia nisi venenatis tristique. -webnode.com|15|15|15|nhirthe@storify.com|Y|Mauris lacinia sapien quis libero. Nullam sit amet turpis elementum ligula vehicula consequat. Morbi a ipsum. -nature.com|16|16|16|nnussiif@yale.edu|N|Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. -prweb.com|17|17|17||Y|Morbi ut odio. Cras mi pede, malesuada in, imperdiet et, commodo vulputate, justo. In blandit ultrices enim. -economist.com|18|18|18|klarkbyh@usnews.com|N|Nulla suscipit ligula in lacus. -i2i.jp|19|19|19|rharrisi@foxnews.com|N|Ut at dolor quis odio consequat varius. Integer ac leo. -discuz.net|20|20|20|bmeaj@miibeian.gov.cn|N|Morbi quis tortor id nulla ultrices aliquet. -google.de|21|21|21|ubakesefk@4shared.com|Y|Etiam vel augue. Vestibulum rutrum rutrum neque. -trellian.com|22|22|22|ekleimtl@amazon.co.jp|N|Duis bibendum. Morbi non quam nec dui luctus rutrum. Nulla tellus. -hhs.gov|23|23|23|mbulmanm@nymag.com|Y|Vestibulum rutrum rutrum neque. -whitehouse.gov|24|24|24|cstudden@fc2.com|N|Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus. Phasellus in felis. -cbc.ca|25|25|25|kbuskeo@shutterfly.com|Y| -prweb.com|26|26|26|hcoldbathp@reddit.com|N|Nullam varius. Nulla facilisi. Cras non velit nec nisi vulputate nonummy. -wunderground.com|27|27|27|nmessinghamq@macromedia.com|Y|Aliquam erat volutpat. In congue. -netlog.com|28|28|28|rthawr@ow.ly|Y|Nunc purus. Phasellus in felis. -biblegateway.com|29|29|29|whurlstons@github.com|Y| -istockphoto.com|30|30|30|mjiroutekt@un.org|N|Pellentesque eget nunc. Donec quis orci eget orci vehicula condimentum. Curabitur in libero ut massa volutpat convallis. -nyu.edu|31|31|31|kpatmoreu@hhs.gov|N|Morbi odio odio, elementum eu, interdum eu, tincidunt in, leo. Maecenas pulvinar lobortis est. Phasellus sit amet erat. -github.io|32|32|32|tgaberv@businessweek.com|N|Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. -globo.com|33|33|33|tmidlarw@google.com.br|N|Maecenas rhoncus aliquam lacus. -constantcontact.com|34|34|34|plaverenzx@cdbaby.com|Y|Aenean lectus. Pellentesque eget nunc. Donec quis orci eget orci vehicula condimentum. -howstuffworks.com|35|35|35|agermainy@bloomberg.com|N|Nullam molestie nibh in lectus. Pellentesque at nulla. Suspendisse potenti. -stanford.edu|36|36|36|ndabornz@smh.com.au|Y|Cras non velit nec nisi vulputate nonummy. Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. -csmonitor.com|37|37|37||Y|Nulla justo. Aliquam quis turpis eget elit sodales scelerisque. -dagondesign.com|38|38|38|emiller11@adobe.com|Y|Donec odio justo, sollicitudin ut, suscipit a, feugiat et, eros. Vestibulum ac est lacinia nisi venenatis tristique. -macromedia.com|39|39|39|bjosephov12@youtube.com|Y|Nulla ac enim. In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. -virginia.edu|40|40|40|bashbe13@xinhuanet.com|Y|Proin risus. -wsj.com|41|41|41||Y|Praesent lectus. Vestibulum quam sapien, varius ut, blandit non, interdum in, ante. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Duis faucibus accumsan odio. -dedecms.com|42|42|42|brain15@sourceforge.net|Y|Sed ante. -ovh.net|43|43|43|gbrehat16@livejournal.com|N|Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. -nyu.edu|44|44|44|fguarnier17@weebly.com|N|Nullam orci pede, venenatis non, sodales sed, tincidunt eu, felis. -kickstarter.com|45|45|45|byes18@google.pl|Y|Pellentesque eget nunc. -about.com|46|46|46|tweine19@wikispaces.com|Y|Integer ac neque. Duis bibendum. -prlog.org|47|47|47|cweeden1a@cocolog-nifty.com|N|Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Donec pharetra, magna vestibulum aliquet ultrices, erat tortor sollicitudin mi, sit amet lobortis sapien sapien non mi. Integer ac neque. -sciencedaily.com|48|48|48|wforcade1b@sciencedaily.com|N|Etiam justo. Etiam pretium iaculis justo. -plala.or.jp|49|49|49|gcarleton1c@quantcast.com|Y|Phasellus sit amet erat. Nulla tempus. -psu.edu|50|50|50|jabbati1d@omniture.com|N|Morbi non quam nec dui luctus rutrum. Nulla tellus. In sagittis dui vel nisl. +Anomaly.gov|1|1|1|ggennrich0@utexas.edu|N|Praesent id massa id nisl venenatis lacinia. +TestDomain.gov|2|2|2|lrome1@uol.com.br|Y|In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. +FakeWebsite1.GOV|3|3|3|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. +FakeWebsite2.GOV|4|4|4|plarderot3@t.co|Y|Morbi quis tortor id nulla ultrices aliquet. Maecenas leo odio, condimentum id, luctus nec, molestie sed, justo. Pellentesque viverra pede ac diam. \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index af1fc35f7..6b6be3400 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -5,4 +5,4 @@ FakeWebsite1|USER2|tech FakeWebsite1|USER3|billing FakeWebsite2.GOV|USER4|admin FakeWebsite2.GOV|USER5|billing -FakeWebsite2.GOV|USER6|tech +FakeWebsite2.GOV|USER6|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index a2c247e24..bf74b0662 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,4 +1,4 @@ Anomaly.gov|muahaha| TestDomain.gov|ok| FakeWebsite1.GOV|serverHold| -FakeWebsite2.GOV|Hold| +FakeWebsite2.GOV|Hold| \ No newline at end of file diff --git a/src/registrar/tests/data/test_organization_adhoc.txt b/src/registrar/tests/data/test_organization_adhoc.txt index ecab84a6a..4baf76a50 100644 --- a/src/registrar/tests/data/test_organization_adhoc.txt +++ b/src/registrar/tests/data/test_organization_adhoc.txt @@ -2,50 +2,4 @@ orgid|orgname|orgstreet|orgcity|orgstate|orgzip|orgcountrycode 1|Flashdog|298 Monument Hill|Lakeland|Florida|33805|US 2|Gigaclub|782 Mosinee Lane|Alexandria|Louisiana|71307|US 3|Midel|376 Donald Pass|Waco|Texas|76705|US -4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US -5|Kwideo|01 Lotheville Place|Beaumont|Texas|77713|US -6|Brainverse|8 Jenifer Point|Fort Myers|Florida|33994|US -7|Brainsphere|74264 Reinke Place|Flint|Michigan|48550|US -8|Pixoboo|40090 Lillian Avenue|Metairie|Louisiana|70033|US -9|Topicshots|99331 Quincy Alley|Cleveland|Ohio|44177|US -10|Eayo|17364 Vahlen Avenue|El Paso|Texas|88558|US -11|Myworks|961 Kim Park|Honolulu|Hawaii|96845|US -12|Flashset|262 Mcguire Parkway|Rochester|New York|14683|US -13|Quatz|8 Forest Street|Warren|Ohio|44485|US -14|Kazio|928 Carey Plaza|Miami|Florida|33196|US -15|DabZ|05350 Claremont Circle|Lexington|Kentucky|40581|US -16|Livepath|698 5th Crossing|Boca Raton|Florida|33499|US -17|Centimia|82 Packers Court|Simi Valley|California|93094|US -18|Avavee|4 Old Gate Center|Tucson|Arizona|85710|US -19|Wikizz|74785 Oak Valley Crossing|Phoenix|Arizona|85040|US -20|Wikivu|877 Gulseth Park|Tallahassee|Florida|32309|US -21|Brainbox|8 Esker Lane|Lexington|Kentucky|40524|US -22|Jaxworks|2 Prairieview Street|Young America|Minnesota|55573|US -23|Youfeed|191 Ramsey Junction|Suffolk|Virginia|23436|US -24|Ntags|24 Melby Court|Kansas City|Missouri|64136|US -25|Realblab|119 Butternut Avenue|Dallas|Texas|75323|US -26|Trudeo|69 Cordelia Park|Palmdale|California|93591|US -27|Wordware|74540 Jenifer Pass|Lake Charles|Louisiana|70607|US -28|Jaxnation|80974 Homewood Avenue|Philadelphia|Pennsylvania|19160|US -29|Latz|01989 Red Cloud Hill|Columbus|Ohio|43226|US -30|Fivespan|0 Ryan Plaza|Honolulu|Hawaii|96805|US -31|Youfeed|24930 Hoard Park|San Antonio|Texas|78260|US -32|Browsetype|49 Waxwing Circle|Oklahoma City|Oklahoma|73119|US -33|Oba|8426 Thompson Parkway|Anaheim|California|92825|US -34|Yodo|64815 Thackeray Crossing|Salinas|California|93907|US -35|Thoughtstorm|79 Del Sol Drive|Evansville|Indiana|47719|US -36|Yamia|2 Marquette Junction|Newark|New Jersey|07195|US -37|Demimbu|14 American Ash Trail|Bronx|New York|10474|US -38|Rhybox|6234 Cambridge Drive|Fort Lauderdale|Florida|33305|US -39|Fivespan|4 Fair Oaks Terrace|Phoenix|Arizona|85040|US -40|Skipfire|4 Dayton Circle|Mesquite|Texas|75185|US -41|Thoughtmix|52 Claremont Avenue|Indianapolis|Indiana|46207|US -42|Meembee|59 Green Ridge Park|Gainesville|Georgia|30506|US -43|Trudeo|54 Schurz Place|Chicago|Illinois|60609|US -44|Tazz|722 Bunker Hill Place|Bronx|New York|10454|US -45|Skiba|8776 Pennsylvania Way|Fayetteville|North Carolina|28314|US -46|Zoomlounge|56 Sloan Circle|Evansville|Indiana|47712|US -47|Cogilith|7 American Ash Trail|Houston|Texas|77255|US -48|Browsebug|15903 Stephen Hill|Arlington|Virginia|22244|US -49|Yamia|9144 Graedel Crossing|Lehigh Acres|Florida|33972|US -501|Lazzy|4958 Kensington Alley|Fayetteville|North Carolina|28305|US +4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index fbe400692..7d6482db8 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -54,6 +54,13 @@ class TestMigrations(TestCase): f"{self.test_data_file_location}/{self.test_domain_contact_filename}", f"{self.test_data_file_location}/{self.test_contact_filename}", f"{self.test_data_file_location}/{self.test_domain_status_filename}", + directory=self.test_data_file_location, + agency_adhoc_filename=self.test_agency_adhoc_filename, + domain_additional_filename=self.test_domain_additional, + domain_escrow_filename=self.test_escrow_domains_daily, + domain_adhoc_filename=self.test_domain_types_adhoc, + organization_adhoc_filename=self.test_organization_adhoc, + authority_adhoc_filename=self.test_authority_adhoc_filename, ) def run_transfer_domains(self): @@ -185,7 +192,8 @@ class TestMigrations(TestCase): expected_missing_domain_invitations, ) - def test_load_transition_domain(self): + def test_load_empty_transition_domain(self): + """Loads TransitionDomains without additional data""" self.run_load_domains() # STEP 2: (analyze the tables just like the migration @@ -209,6 +217,9 @@ class TestMigrations(TestCase): expected_missing_domain_informations, expected_missing_domain_invitations, ) + + def test_load_full_transition_domain(self): + pass def test_transfer_transition_domains_to_domains(self): # TODO: setup manually instead of calling other script From 4e22e26065204fa4c941fbe3a885c01a951fab07 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:16:57 -0700 Subject: [PATCH 39/88] Bug fix --- .../commands/utility/extra_transition_domain_helper.py | 4 ++-- .../commands/utility/transition_domain_arguments.py | 8 +++++++- src/registrar/tests/test_transition_domain_migrations.py | 3 --- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index e98954c60..496ed87f1 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -136,9 +136,9 @@ class LoadExtraTransitionDomain: def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() - arguments = options.args_extra_transition_domain() + print(f"options correct? {options.agency_adhoc_filename}") # Reads and parses migration files - self.parsed_data_container = ExtraTransitionDomain(**arguments) + self.parsed_data_container = ExtraTransitionDomain(options) self.parsed_data_container.parse_all_files(options.infer_filenames) def update_transition_domain_models(self): diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index e95425255..18bb99628 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -6,6 +6,11 @@ from registrar.management.commands.utility.epp_data_containers import EnumFilena @dataclass class TransitionDomainArguments: """Stores arguments for load_transition_domain""" + def __init__(self, **kwargs): + self.kwargs = kwargs + for k, v in kwargs.items(): + if hasattr(self, k): + setattr(self, k, v) # Settings # directory: Optional[str] = field(default="migrationdata", repr=True) @@ -20,7 +25,8 @@ class TransitionDomainArguments: authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) ## Data files ## - domain_additional_filename: Optional[str] = field(default=None, repr=True) + domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True) + domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) domain_contacts_filename: Optional[str] = field(default=None, repr=True) domain_statuses_filename: Optional[str] = field(default=None, repr=True) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 7d6482db8..cdad79ed8 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -14,9 +14,6 @@ from django.core.management import call_command from unittest.mock import patch class TestMigrations(TestCase): - - """ """ - def setUp(self): """ """ # self.load_transition_domain_script = "load_transition_domain", From 44c7631eaa87eefa41b66bd85bf2568bf3f80534 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:45:14 -0700 Subject: [PATCH 40/88] Test data and test changes --- .../commands/load_transition_domain.py | 7 +-- .../transfer_transition_domains_to_domains.py | 2 +- .../utility/extra_transition_domain_helper.py | 15 +++--- .../utility/transition_domain_arguments.py | 19 ++++++- src/registrar/models/transition_domain.py | 16 +++++- .../tests/data/test_agency_adhoc.txt | 3 +- .../tests/data/test_authority_adhoc.txt | 3 +- src/registrar/tests/data/test_contacts.txt | 5 +- .../tests/data/test_domain_additional.txt | 7 +-- .../tests/data/test_domain_contacts.txt | 15 +++--- .../tests/data/test_domain_statuses.txt | 5 +- .../tests/data/test_escrow_domains_daily.txt | 5 ++ .../tests/data/test_organization_adhoc.txt | 5 +- .../test_transition_domain_migrations.py | 53 ++++++++++++++++++- 14 files changed, 131 insertions(+), 29 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 1b3cf8642..d08b1c2ed 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -72,7 +72,9 @@ class Command(BaseCommand): "--directory", default="migrationdata", help="Desired directory" ) parser.add_argument( - "--infer_filenames", default=False, help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." + "--infer_filenames", + action=argparse.BooleanOptionalAction, + help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." ) parser.add_argument( "--agency_adhoc_filename", @@ -86,10 +88,9 @@ class Command(BaseCommand): ) parser.add_argument( "--domain_escrow_filename", - default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], + default=EnumFilenames.DOMAIN_ESCROW.value[1], help="Defines the filename for creation/expiration domain data", ) - #domain_escrow_filename parser.add_argument( "--domain_adhoc_filename", default=EnumFilenames.DOMAIN_ADHOC.value[1], diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 7ed0ba009..e58d7dc24 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -371,7 +371,7 @@ class Command(BaseCommand): org_type = transition_domain.organization_type fed_type = transition_domain.federal_type fed_agency = transition_domain.federal_agency - + match org_type: case "Federal": org_type = ("federal", "Federal") diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 496ed87f1..7c59c9f76 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -151,7 +151,7 @@ class LoadExtraTransitionDomain: updated_transition_domains = [] failed_transition_domains = [] for transition_domain in all_transition_domains: - domain_name = transition_domain.domain_name.upper() + domain_name = transition_domain.domain_name updated_transition_domain = transition_domain try: # STEP 1: Parse organization data @@ -188,6 +188,7 @@ class LoadExtraTransitionDomain: # If we run into an exception on this domain, # Just skip over it and log that it happened. + # Q: Should we just throw an exception? except Exception as err: logger.debug(err) logger.error( @@ -511,6 +512,8 @@ class LoadExtraTransitionDomain: def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, based off a desired_id""" + l = self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id.lower()) + print(f"is it happening here? {l} for id {desired_id}") return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: @@ -889,11 +892,6 @@ class ExtraTransitionDomain: def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) - """ - for row in reader: - print({key: type(key) for key in row.keys()}) # print out the keys and their types - test = {row[id_field]: dataclass_type(**row)} - """ dict_data = {} for row in reader: if None in row: @@ -903,6 +901,11 @@ class ExtraTransitionDomain: print(f"key: {key} value: {value}") continue row_id = row[id_field] + + # To maintain pairity with the load_transition_domain + # script, we store this data in lowercase. + if id_field == "domainname" and row_id is not None: + row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 18bb99628..70f13cfb5 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -5,13 +5,30 @@ from registrar.management.commands.utility.epp_data_containers import EnumFilena @dataclass class TransitionDomainArguments: - """Stores arguments for load_transition_domain""" + """Stores arguments for load_transition_domain, structurally a mix + of a dataclass and a regular class, meaning we get a hardcoded + representation of the values we want, while maintaining flexiblity + and reducing boilerplate. + + All pre-defined fields are optional but will remain on the model definition. + In this event, they are provided a default value if none is given. + """ + + # Maintains an internal kwargs list and sets values + # that match the class definition. def __init__(self, **kwargs): self.kwargs = kwargs for k, v in kwargs.items(): if hasattr(self, k): setattr(self, k, v) + # These all use field() to minimize typing and/or lambda. + # Since this file is bound to expand, we can save time + # by reducing the line count from 2-3 to just 1 line + # each time we want to add a new filename or option. + + # This approach is also used in EppLib internally for similar reasons. + # Settings # directory: Optional[str] = field(default="migrationdata", repr=True) sep: Optional[str] = field(default="|", repr=True) diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index a31b27e7e..11b0ccff8 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -79,6 +79,20 @@ class TransitionDomain(TimeStampedModel): ) def __str__(self): + + return f""" + username="{self.username}", + domain_name="{self.domain_name}", + status="{self.status}", + email_sent={self.email_sent}, + organization_type="{self.organization_type}", + organization_name="{self.organization_name}", + federal_type="{self.federal_type}", + federal_agency="{self.federal_agency}", + epp_creation_date={self.epp_creation_date}, + epp_expiration_date={self.epp_expiration_date} + """ + """ return ( f"\n-----TRANSITION DOMAIN------\n" f"domainName: {self.domain_name}, \n" @@ -91,4 +105,4 @@ class TransitionDomain(TimeStampedModel): f"federal_agency: {self.federal_agency}, \n" f"epp_creation_date: {self.epp_creation_date}, \n" f"epp_expiration_date: {self.epp_expiration_date}, \n" - ) + )""" diff --git a/src/registrar/tests/data/test_agency_adhoc.txt b/src/registrar/tests/data/test_agency_adhoc.txt index 3de43d2d9..0ff33b852 100644 --- a/src/registrar/tests/data/test_agency_adhoc.txt +++ b/src/registrar/tests/data/test_agency_adhoc.txt @@ -2,4 +2,5 @@ agencyid|agencyname|active|isfederal 1|Thoughtstorm|N|Y 2|Minyx|Y|N 3|Demivee|N|Y -4|InnoZ|Y|Y \ No newline at end of file +4|InnoZ|Y|Y +5|igorville|Y|N \ No newline at end of file diff --git a/src/registrar/tests/data/test_authority_adhoc.txt b/src/registrar/tests/data/test_authority_adhoc.txt index aa24e9385..01e1a97db 100644 --- a/src/registrar/tests/data/test_authority_adhoc.txt +++ b/src/registrar/tests/data/test_authority_adhoc.txt @@ -2,4 +2,5 @@ authorityid|firstname|middlename|lastname|email|phonenumber|agencyid|addlinfo 1|Gregoor|middle|Kalinke|gkalinke0@indiegogo.com|(773) 172-5515|1|Asparagus - Mexican 2|Fayre||Filippozzi|ffilippozzi1@hugedomains.com|(357) 487-4280|2|Steampan - Foil 3|Gabey||Lightbody|glightbody2@fc2.com|(332) 816-5691|3|Soup - Campbells, Minestrone -4|Seline||Tower|stower3@answers.com|(151) 539-6028|4|Kiwi Gold Zespri \ No newline at end of file +4|Seline||Tower|stower3@answers.com|(151) 539-6028|4|Kiwi Gold Zespri +5|Joe||Smoe|joe@smoe.gov|(111) 111-1111|5|Kiwi Gold Zespri \ No newline at end of file diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt index 89f57ccf8..725028d6b 100644 --- a/src/registrar/tests/data/test_contacts.txt +++ b/src/registrar/tests/data/test_contacts.txt @@ -4,4 +4,7 @@ USER2|12355_CONTACT|123-123-1234||918-000-0000||susy.martin4@test.com|GSA|SOMECO USER3|12356_CONTACT|123-123-1234||918-000-0000||stephania.winters4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T18:28:09Z| USER4|12357_CONTACT|123-123-1234||918-000-0000||alexandra.bobbitt5@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-08-02T22:13:09Z| USER5|12362_CONTACT|123-123-1234||918-000-0000||jospeh.mcdowell3@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:33:09Z| -USER6|12363_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file +USER6|12363_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER7|12364_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER8|12365_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER9|12366_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_additional.txt b/src/registrar/tests/data/test_domain_additional.txt index b8194daa8..7257de079 100644 --- a/src/registrar/tests/data/test_domain_additional.txt +++ b/src/registrar/tests/data/test_domain_additional.txt @@ -1,5 +1,6 @@ domainname|domaintypeid|authorityid|orgid|securitycontactemail|dnsseckeymonitor|domainpurpose Anomaly.gov|1|1|1|ggennrich0@utexas.edu|N|Praesent id massa id nisl venenatis lacinia. -TestDomain.gov|2|2|2|lrome1@uol.com.br|Y|In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. -FakeWebsite1.GOV|3|3|3|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. -FakeWebsite2.GOV|4|4|4|plarderot3@t.co|Y|Morbi quis tortor id nulla ultrices aliquet. Maecenas leo odio, condimentum id, luctus nec, molestie sed, justo. Pellentesque viverra pede ac diam. \ No newline at end of file +TESTDOMAIN.GOV|2|2|2|lrome1@uol.com.br|Y|In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. +FakeWebsite1.gov|3|3|3|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. +FakeWebsite2.gov|4|4|4|plarderot3@t.co|Y|Morbi quis tortor id nulla ultrices aliquet. Maecenas leo odio, condimentum id, luctus nec, molestie sed, justo. Pellentesque viverra pede ac diam. +FakeWebsite3.gov|13|5|5|ybrommage2@vistaprint.com|Y|In hac habitasse platea dictumst. \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index 6b6be3400..4ffce8046 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -1,8 +1,11 @@ Anomaly.gov|ANOMALY|tech TestDomain.gov|TESTUSER|admin -FakeWebsite1|USER1|admin -FakeWebsite1|USER2|tech -FakeWebsite1|USER3|billing -FakeWebsite2.GOV|USER4|admin -FakeWebsite2.GOV|USER5|billing -FakeWebsite2.GOV|USER6|tech \ No newline at end of file +FakeWebsite1.gov|USER1|admin +FakeWebsite1.gov|USER2|tech +FakeWebsite1.gov|USER3|billing +FakeWebsite2.gov|USER4|admin +FakeWebsite2.gov|USER5|billing +FakeWebsite2.gov|USER6|tech +FakeWebsite3.gov|USER7|admin +FakeWebsite3.gov|USER8|billing +FakeWebsite3.gov|USER9|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index bf74b0662..a5377f5fd 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,4 +1,5 @@ Anomaly.gov|muahaha| TestDomain.gov|ok| -FakeWebsite1.GOV|serverHold| -FakeWebsite2.GOV|Hold| \ No newline at end of file +FakeWebsite1.gov|serverHold| +FakeWebsite2.gov|Hold| +FakeWebsite3.gov|ok| \ No newline at end of file diff --git a/src/registrar/tests/data/test_escrow_domains_daily.txt b/src/registrar/tests/data/test_escrow_domains_daily.txt index e69de29bb..0f5c15553 100644 --- a/src/registrar/tests/data/test_escrow_domains_daily.txt +++ b/src/registrar/tests/data/test_escrow_domains_daily.txt @@ -0,0 +1,5 @@ +Anomaly.gov|SOME_STRING||data|data|data|data|2008-03-09T16:12:47Z|DATA2|ctldbatch|2022-06-06T01:33:10Z|2023-03-09T16:12:47Z|2023-02-09T16:12:47Z +TestDomain.gov|SOME_STRING|data|data|data|data|data|2014-03-15T15:45:05Z|DATA2|ctldbatch|2022-02-13T17:33:07Z|2023-03-15T15:45:05Z|2023-02-15T15:45:05Z +FakeWebsite1.gov|SOME_STRING||data|data|data|data|2020-06-14T16:30:06Z|DATA2|ctldbatch|2022-05-16T14:58:10Z|2023-06-14T16:30:06Z|2023-05-14T16:30:06Z +FakeWebsite2.gov|SOME_STRING||data|data|data|data|2004-05-07T04:00:00Z|DATA2|ctldbatch|2022-08-18T15:23:09Z|2023-09-30T18:37:39Z|2023-08-30T18:37:39Z +FakeWebsite3.gov|SOME_STRING||data|data|data|data|2004-05-07T04:00:00Z|DATA2|ctldbatch|2022-08-18T15:23:09Z|2023-09-30T18:37:39Z|2023-08-30T18:37:39Z \ No newline at end of file diff --git a/src/registrar/tests/data/test_organization_adhoc.txt b/src/registrar/tests/data/test_organization_adhoc.txt index 4baf76a50..a39361a10 100644 --- a/src/registrar/tests/data/test_organization_adhoc.txt +++ b/src/registrar/tests/data/test_organization_adhoc.txt @@ -1,5 +1,6 @@ orgid|orgname|orgstreet|orgcity|orgstate|orgzip|orgcountrycode 1|Flashdog|298 Monument Hill|Lakeland|Florida|33805|US 2|Gigaclub|782 Mosinee Lane|Alexandria|Louisiana|71307|US -3|Midel|376 Donald Pass|Waco|Texas|76705|US -4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US \ No newline at end of file +3|Midel|376 Joe Pass|Waco|Texas|76705|US +4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US +5|Sushi|9999 Sushi Way|Columbus|Ohio|43268|US \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index cdad79ed8..1740530b1 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -216,7 +216,58 @@ class TestMigrations(TestCase): ) def test_load_full_transition_domain(self): - pass + # Load command + self.run_load_domains() + + # We should get a consistent number + # of records + expected_total_transition_domains = 8 + expected_total_domains = 0 + expected_total_domain_informations = 0 + expected_total_domain_invitations = 0 + + expected_missing_domains = 8 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 8 + expected_missing_domain_invitations = 8 + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + expected_transition_domains = [ + TransitionDomain( + username="", + domain_name="anomaly.gov", + status="ready", + email_sent=False, + organization_type="None", + organization_name="Flashdog", + federal_type="None", + federal_agency="None", + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="", + domain_name="anomaly.gov", + status="ready", + email_sent=False, + organization_type="None", + organization_name="Flashdog", + federal_type="None", + federal_agency="None", + epp_creation_date=None, + epp_expiration_date=None + ), + ] + # Afterwards, their values should be what we expect def test_transfer_transition_domains_to_domains(self): # TODO: setup manually instead of calling other script From f7d906245e9b6a5c2574fbc54f4d3030ee70035a Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:19:07 -0700 Subject: [PATCH 41/88] Finish test / Add debug flag --- .../utility/extra_transition_domain_helper.py | 11 +- .../test_transition_domain_migrations.py | 130 +++++++++++++++--- 2 files changed, 122 insertions(+), 19 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7c59c9f76..3583e9af1 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -136,7 +136,7 @@ class LoadExtraTransitionDomain: def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() - print(f"options correct? {options.agency_adhoc_filename}") + self.debug = options.debug # Reads and parses migration files self.parsed_data_container = ExtraTransitionDomain(options) self.parsed_data_container.parse_all_files(options.infer_filenames) @@ -178,8 +178,9 @@ class LoadExtraTransitionDomain: #if updated_transition_domain.__dict__ != transition_domain.__dict__: updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) - - self.parse_logs.display_logs_by_domain_name(domain_name) + if self.debug: + # Display errors for this specific domain + self.parse_logs.display_logs_by_domain_name(domain_name) logger.info( f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" @@ -198,6 +199,10 @@ class LoadExtraTransitionDomain: ) failed_transition_domains.append(domain_name) + if self.debug: + # Display misc errors (not associated to a domain) + self.parse_logs.display_logs_by_domain_name(None) + failed_count = len(failed_transition_domains) if failed_count == 0: logger.info( diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 1740530b1..4d6f043b6 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -168,7 +168,7 @@ class TestMigrations(TestCase): # STEP 2: (analyze the tables just like the # migration script does, but add assert statements) - expected_total_transition_domains = 8 + expected_total_transition_domains = 9 expected_total_domains = 4 expected_total_domain_informations = 4 expected_total_domain_invitations = 7 @@ -195,7 +195,7 @@ class TestMigrations(TestCase): # STEP 2: (analyze the tables just like the migration # script does, but add assert statements) - expected_total_transition_domains = 8 + expected_total_transition_domains = 9 expected_total_domains = 0 expected_total_domain_informations = 0 expected_total_domain_invitations = 0 @@ -221,15 +221,15 @@ class TestMigrations(TestCase): # We should get a consistent number # of records - expected_total_transition_domains = 8 + expected_total_transition_domains = 9 expected_total_domains = 0 expected_total_domain_informations = 0 expected_total_domain_invitations = 0 - expected_missing_domains = 8 + expected_missing_domains = 9 expected_duplicate_domains = 0 - expected_missing_domain_informations = 8 - expected_missing_domain_invitations = 8 + expected_missing_domain_informations = 9 + expected_missing_domain_invitations = 9 self.compare_tables( expected_total_transition_domains, expected_total_domains, @@ -247,27 +247,125 @@ class TestMigrations(TestCase): domain_name="anomaly.gov", status="ready", email_sent=False, - organization_type="None", + organization_type=None, organization_name="Flashdog", - federal_type="None", - federal_agency="None", + federal_type=None, + federal_agency=None, epp_creation_date=None, epp_expiration_date=None ), TransitionDomain( - username="", - domain_name="anomaly.gov", + username="testuser@gmail.com", + domain_name="testdomain.gov", status="ready", email_sent=False, - organization_type="None", - organization_name="Flashdog", - federal_type="None", - federal_agency="None", + organization_type=None, + organization_name="Gigaclub", + federal_type=None, + federal_agency=None, epp_creation_date=None, epp_expiration_date=None ), + TransitionDomain( + username="agustina.wyman7@test.com", + domain_name="fakewebsite1.gov", + status="on hold", + email_sent=False, + organization_type=None, + organization_name="Midel", + federal_type=None, + federal_agency=None, + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="susy.martin4@test.com", + domain_name="fakewebsite1.gov", + status="on hold", + email_sent=False, + organization_type=None, + organization_name="Midel", + federal_type=None, + federal_agency=None, + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="stephania.winters4@test.com", + domain_name="fakewebsite1.gov", + status="on hold", + email_sent=False, + organization_type=None, + organization_name="Midel", + federal_type=None, + federal_agency=None, + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="alexandra.bobbitt5@test.com", + domain_name="fakewebsite2.gov", + status="on hold", + email_sent=False, + organization_type="Federal", + organization_name="Fanoodle", + federal_type="Executive", + federal_agency="InnoZ", + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="jospeh.mcdowell3@test.com", + domain_name="fakewebsite2.gov", + status="on hold", + email_sent=False, + organization_type="Federal", + organization_name="Fanoodle", + federal_type="Executive", + federal_agency="InnoZ", + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="reginald.ratcliff4@test.com", + domain_name="fakewebsite2.gov", + status="on hold", + email_sent=False, + organization_type="Federal", + organization_name="Fanoodle", + federal_type="Executive", + federal_agency="InnoZ", + epp_creation_date=None, + epp_expiration_date=None + ), + TransitionDomain( + username="reginald.ratcliff4@test.com", + domain_name="fakewebsite3.gov", + status="ready", + email_sent=False, + organization_type="City", + organization_name="Sushi", + federal_type=None, + federal_agency=None, + epp_creation_date=None, + epp_expiration_date=None + ) ] + # Afterwards, their values should be what we expect + all_transition_domains = TransitionDomain.objects.all() + for domain in all_transition_domains: + for expected in expected_transition_domains: + + # This data gets created when the object is, + # so we should just match it. Not relevant + # to the added data. + expected.id = domain.id + expected.created_at = domain.created_at + expected.updated_at = domain.updated_at + + # Each TransitionDomain should have the correct data + self.assertEqual(domain, expected) def test_transfer_transition_domains_to_domains(self): # TODO: setup manually instead of calling other script @@ -275,7 +373,7 @@ class TestMigrations(TestCase): self.run_transfer_domains() # Analyze the tables - expected_total_transition_domains = 8 + expected_total_transition_domains = 9 expected_total_domains = 4 expected_total_domain_informations = 4 expected_total_domain_invitations = 7 From d321b3148631631203bd28bb6274b0ae90c02ff6 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 13:35:40 -0600 Subject: [PATCH 42/88] Added missing agencies --- src/registrar/models/domain_application.py | 69 +++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index 68fbfab0d..708fceb35 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -167,23 +167,33 @@ class DomainApplication(TimeStampedModel): "Administrative Conference of the United States", "Advisory Council on Historic Preservation", "American Battle Monuments Commission", + "AMTRAK", "Appalachian Regional Commission", ( "Appraisal Subcommittee of the Federal Financial " "Institutions Examination Council" ), + "Appraisal Subcommittee", + "Architect of the Capitol", "Armed Forces Retirement Home", + "Barry Goldwater Scholarship and Excellence in Education Foundation", "Barry Goldwater Scholarship and Excellence in Education Program", "Central Intelligence Agency", + "Chemical Safety Board", "Christopher Columbus Fellowship Foundation", + "Civil Rights Cold Case Records Review Board", "Commission for the Preservation of America's Heritage Abroad", "Commission of Fine Arts", "Committee for Purchase From People Who Are Blind or Severely Disabled", "Commodity Futures Trading Commission", + "Congressional Budget Office", "Consumer Financial Protection Bureau", "Consumer Product Safety Commission", + "Corporation for National & Community Service", "Corporation for National and Community Service", "Council of Inspectors General on Integrity and Efficiency", + "Court Services and Offender Supervision", + "Cyberspace Solarium Commission", "DC Court Services and Offender Supervision Agency", "DC Pre-trial Services", "Defense Nuclear Facilities Safety Board", @@ -209,12 +219,15 @@ class DomainApplication(TimeStampedModel): "Election Assistance Commission", "Environmental Protection Agency", "Equal Employment Opportunity Commission", + "Executive Office of the President", "Export-Import Bank of the United States", + "Export/Import Bank of the U.S.", "Farm Credit Administration", "Farm Credit System Insurance Corporation", "Federal Communications Commission", "Federal Deposit Insurance Corporation", "Federal Election Commission", + "Federal Energy Regulatory Commission", "Federal Financial Institutions Examination Council", "Federal Housing Finance Agency", "Federal Judiciary", @@ -222,50 +235,74 @@ class DomainApplication(TimeStampedModel): "Federal Maritime Commission", "Federal Mediation and Conciliation Service", "Federal Mine Safety and Health Review Commission", + "Federal Permitting Improvement Steering Council", + "Federal Reserve Board of Governors", "Federal Reserve System", "Federal Trade Commission", "General Services Administration", + "gov Administration", + "Government Accountability Office", + "Government Publishing Office", "Gulf Coast Ecosystem Restoration Council", "Harry S Truman Scholarship Foundation", + "Harry S. Truman Scholarship Foundation", + "Institute of Museum and Library Services", "Institute of Peace", "Inter-American Foundation", "International Boundary and Water Commission: United States and Mexico", - "International Boundary Commission: United States and Canada", - "International Joint Commission: United States and Canada", + "International Boundary Commission: United States and Canada", + "International Joint Commission: United States and Canada", "James Madison Memorial Fellowship Foundation", "Japan-United States Friendship Commission", + "Japan-US Friendship Commission", + "John F. Kennedy Center for Performing Arts", "John F. Kennedy Center for the Performing Arts", "Legal Services Corporation", "Legislative Branch", + "Library of Congress", "Marine Mammal Commission", + "Medicaid and CHIP Payment and Access Commission", + "Medical Payment Advisory Commission", "Medicare Payment Advisory Commission", "Merit Systems Protection Board", "Millennium Challenge Corporation", + "Morris K. Udall and Stewart L. Udall Foundation", "National Aeronautics and Space Administration", "National Archives and Records Administration", "National Capital Planning Commission", "National Council on Disability", "National Credit Union Administration", + "National Endowment for the Arts", + "National Endowment for the Humanities", "National Foundation on the Arts and the Humanities", "National Gallery of Art", + "National Indian Gaming Commission", "National Labor Relations Board", "National Mediation Board", "National Science Foundation", + "National Security Commission on Artificial Intelligence", "National Transportation Safety Board", + "Networking Information Technology Research and Development", + "Non-Federal Agency", "Northern Border Regional Commission", "Nuclear Regulatory Commission", "Nuclear Safety Oversight Committee", "Nuclear Waste Technical Review Board", + "Occupational Safety & Health Review Commission", "Occupational Safety and Health Review Commission", "Office of Compliance", + "Office of Congressional Workplace Rights", "Office of Government Ethics", "Office of Navajo and Hopi Indian Relocation", "Office of Personnel Management", + "Open World Leadership Center", "Overseas Private Investment Corporation", "Peace Corps", "Pension Benefit Guaranty Corporation", "Postal Regulatory Commission", + "Presidio Trust", "Privacy and Civil Liberties Oversight Board", + "Public Buildings Reform Board", "Public Defender Service for the District of Columbia", "Railroad Retirement Board", "Securities and Exchange Commission", @@ -273,30 +310,58 @@ class DomainApplication(TimeStampedModel): "Small Business Administration", "Smithsonian Institution", "Social Security Administration", + "Social Security Advisory Board", + "Southeast Crescent Regional Commission", + "Southwest Border Regional Commission", "State Justice Institute", "State, Local, and Tribal Government", "Stennis Center for Public Service", "Surface Transportation Board", "Tennessee Valley Authority", "The Executive Office of the President", + "The Intelligence Community", + "The Legislative Branch", + "The Supreme Court", + "The United States World War One Centennial Commission", "U.S. Access Board", "U.S. Agency for Global Media", "U.S. Agency for International Development", + "U.S. Capitol Police", "U.S. Chemical Safety Board", "U.S. China Economic and Security Review Commission", + "U.S. Commission for the Preservation of Americas Heritage Abroad", + "U.S. Commission of Fine Arts", "U.S. Commission on Civil Rights", "U.S. Commission on International Religious Freedom", + "U.S. Courts", + "U.S. Department of Agriculture", "U.S. Interagency Council on Homelessness", "U.S. International Trade Commission", + "U.S. Nuclear Waste Technical Review Board", "U.S. Office of Special Counsel", + "U.S. Peace Corps", "U.S. Postal Service", + "U.S. Semiquincentennial Commission", "U.S. Trade and Development Agency", + "U.S.-China Economic and Security Review Commission", "Udall Foundation", + "United States AbilityOne", + "United States Access Board", "United States African Development Foundation", + "United States Agency for Global Media", "United States Arctic Research Commission", + "United States Global Change Research Program", "United States Holocaust Memorial Museum", + "United States Institute of Peace", + "United States Interagency Council on Homelessness", + "United States International Development Finance Corporation", + "United States International Trade Commission", + "United States Postal Service", + "United States Senate", + "United States Trade and Development Agency", "Utah Reclamation Mitigation and Conservation Commission", "Vietnam Education Foundation", + "Western Hemisphere Drug Policy Commission", "Woodrow Wilson International Center for Scholars", "World War I Centennial Commission", ] From 6df045673d0c1a37361a7272f9c920f8c812b478 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:49:14 -0700 Subject: [PATCH 43/88] More tests --- .../utility/extra_transition_domain_helper.py | 1 - src/registrar/models/transition_domain.py | 16 +---- .../test_transition_domain_migrations.py | 69 ++++++++++++++++++- 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 3583e9af1..7022944a0 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -518,7 +518,6 @@ class LoadExtraTransitionDomain: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, based off a desired_id""" l = self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id.lower()) - print(f"is it happening here? {l} for id {desired_id}") return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 11b0ccff8..a31b27e7e 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -79,20 +79,6 @@ class TransitionDomain(TimeStampedModel): ) def __str__(self): - - return f""" - username="{self.username}", - domain_name="{self.domain_name}", - status="{self.status}", - email_sent={self.email_sent}, - organization_type="{self.organization_type}", - organization_name="{self.organization_name}", - federal_type="{self.federal_type}", - federal_agency="{self.federal_agency}", - epp_creation_date={self.epp_creation_date}, - epp_expiration_date={self.epp_expiration_date} - """ - """ return ( f"\n-----TRANSITION DOMAIN------\n" f"domainName: {self.domain_name}, \n" @@ -105,4 +91,4 @@ class TransitionDomain(TimeStampedModel): f"federal_agency: {self.federal_agency}, \n" f"epp_creation_date: {self.epp_creation_date}, \n" f"epp_expiration_date: {self.epp_expiration_date}, \n" - )""" + ) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4d6f043b6..ef74f8744 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -366,9 +366,76 @@ class TestMigrations(TestCase): # Each TransitionDomain should have the correct data self.assertEqual(domain, expected) + + def test_load_full_transfer_domain(self): + self.run_load_domains() + self.run_transfer_domains() + + # Analyze the tables + expected_total_transition_domains = 9 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 0 + expected_missing_domain_invitations = 1 + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + expected_domains = [ + Domain( + expiration_date=None, + name="anomaly.gov", + state="ready", + ), + Domain( + expiration_date=None, + name="testdomain.gov", + state="ready", + ), + Domain( + expiration_date=None, + name="fakewebsite1.gov", + state="on hold", + ), + Domain( + expiration_date=None, + name="fakewebsite2.gov", + state="on hold", + ), + Domain( + expiration_date=None, + name="fakewebsite3.gov", + state="ready", + ), + ] + + for domain in Domain.objects.all(): + print(f""" + Domain( + expiration_date={domain.expiration_date}, + name="{domain.name}", + state="{domain.state}", + ), + """ + ) + for expected in expected_domains: + expected.id = domain.id + expected.created_at = domain.created_at + expected.updated_at = domain.updated_at + self.assertEqual(domain, expected) def test_transfer_transition_domains_to_domains(self): - # TODO: setup manually instead of calling other script self.run_load_domains() self.run_transfer_domains() From e885046324917ac4f63c1641d387cddec3a46b84 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:59:53 -0700 Subject: [PATCH 44/88] Revert "Merge branch 'main' into za/additional-data-transferred-domains" This reverts commit a6d5cd877b2379ef386eedc719095fe083218b40, reversing changes made to bf88401327cc1dd2b8b804885e535c96f69180a4. --- docs/operations/data_migration.md | 28 +- .../runbooks/rotate_application_secrets.md | 31 +- src/Pipfile | 1 - src/Pipfile.lock | 504 +++++++++--------- src/registrar/assets/sass/_theme/_admin.scss | 2 +- src/registrar/assets/sass/_theme/_alerts.scss | 6 +- src/registrar/config/settings.py | 9 - .../commands/master_domain_migrations.py | 10 +- src/registrar/templates/base.html | 7 +- .../django/admin/domain_change_list.html | 2 +- src/registrar/utility/csv_export.py | 26 +- src/requirements.txt | 21 +- 12 files changed, 291 insertions(+), 356 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 75bd50a9b..2dd00a39f 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -211,12 +211,11 @@ This will allow Docker to mount the files to a container (under `/app`) for our ## Transition Domains (Part 2) - Running the Migration Scripts +*NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.* ### STEP 1: Load Transition Domains -Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. - -(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and and remove "docker compose run -T app" from the command) +Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` @@ -238,8 +237,6 @@ This will delete all the data in transtion_domain. It is helpful if you want to Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): - -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` @@ -254,11 +251,9 @@ Directs the script to load only the first 100 entries into the table. You can a ### STEP 3: Send Domain invitations -To send invitation emails for every transition domain in the transition domain table, execute the following command: - -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) +To send invitations for every transition domain in the transition domain table, execute the following command: ```shell -docker compose run -T app ./manage.py send_domain_invitations -s +docker compose run -T app send_domain_invitations -s ``` ### STEP 4: Test the results (Run the analyzer script) @@ -268,8 +263,6 @@ This script's main function is to scan the transition domain and domain tables f #### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: - -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` @@ -277,8 +270,6 @@ docker compose run -T app ./manage.py master_domain_migrations --debug #### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: - -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt ``` @@ -287,12 +278,13 @@ docker compose run -T app ./manage.py master_domain_migrations --runMigrations - `--runMigrations` -Runs all scripts (in sequence) for transition domain migrations +A boolean (default to true), which triggers running +all scripts (in sequence) for transition domain migrations `--migrationDirectory` The location of the files used for load_transition_domain migration script. -(default is "migrationdata" (This is the sandbox directory)) +(default is "migrationData" (This is the sandbox directory)) Example Usage: *--migrationDirectory /app/tmp* @@ -318,11 +310,13 @@ Delimiter for the migration scripts to correctly parse the given text files. `--debug` -Activates additional print statements +A boolean (default to true), which activates additional print statements `--prompt` -Activates terminal prompts that allows the user to step through each portion of this script. +A boolean (default to true), which activates terminal prompts +that allows the user to step through each portion of this +script. `--limitParse` diff --git a/docs/operations/runbooks/rotate_application_secrets.md b/docs/operations/runbooks/rotate_application_secrets.md index 78c402efe..e91e8427e 100644 --- a/docs/operations/runbooks/rotate_application_secrets.md +++ b/docs/operations/runbooks/rotate_application_secrets.md @@ -76,24 +76,16 @@ These are the client certificate and its private key used to identify the regist The private key is protected by a passphrase for safer transport and storage. -These were generated with the following steps: - -### Step 1: Generate an unencrypted private key with a named curve +These were generated with: ```bash -openssl ecparam -name prime256v1 -genkey -out client_unencrypted.key -``` +openssl genpkey -out client.key \ + -algorithm EC -pkeyopt ec_paramgen_curve:P-256 \ + -aes-256-cbc +openssl req -new -x509 -days 365 \ + -key client.key -out client.crt \ + -subj "/C=US/ST=DC/L=Washington/O=GSA/OU=18F/CN=GOV Prototype Registrar" -### Step 2: Create an encrypted private key with a passphrase - -```bash -openssl pkcs8 -topk8 -v2 aes-256-cbc -in client_unencrypted.key -out client.key -``` - -### Generate the certificate - -```bash -openssl req -new -x509 -days 365 -key client.key -out client.crt -subj "/C=US/ST=DC/L=Washington/O=GSA/OU=18F/CN=GOV Prototype Registrar" ``` (If you can't use openssl on your computer directly, you can access it using Docker as `docker run --platform=linux/amd64 -it --rm -v $(pwd):/apps -w /apps alpine/openssl`.) @@ -105,14 +97,7 @@ base64 client.key base64 client.crt ``` -Note depending on your system you may need to instead run: - -```bash -base64 -i client.key -base64 -i client.crt -``` - -You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vender, make sure to update the kdbx file on Google Drive. +You'll need to give the new certificate to the registry vendor _before_ rotating it in production. ## REGISTRY_HOSTNAME diff --git a/src/Pipfile b/src/Pipfile index d4a9bbafb..33f2c0954 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -6,7 +6,6 @@ name = "pypi" [packages] django = "*" cfenv = "*" -django-cors-headers = "*" pycryptodomex = "*" django-allow-cidr = "*" django-auditlog = "*" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 62eb133dd..9d7daf597 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a0ea45d8f77132e22b8c3437e90903240af1bb29f9994f6ce4ce1e5b8d06b6ed" + "sha256": "423c746438717fb7d281dfac02d3950e6e5033c6190f4adf0c63ccdf9433fae5" }, "pipfile-spec": 6, "requires": {}, @@ -32,29 +32,29 @@ }, "boto3": { "hashes": [ - "sha256:85e2fa361ad3210d30800bad311688261f2673a9b301e0edab56463d89609761", - "sha256:d18688bc5d688decf3cc404430a3ac3ec317be653cdcfbc51104c01f38a66434" + "sha256:38658585791f47cca3fc6aad03838de0136778b533e8c71c6a9590aedc60fbde", + "sha256:a8228522c7db33694c0746dec8b48c05473671626359dd62ab6829eb7871eddc" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.28.76" + "version": "==1.28.66" }, "botocore": { "hashes": [ - "sha256:479abb5a1ee03eb00faa1ea176bc595b2f46f7494777807681a9df45ed99ea18", - "sha256:74e0a4515d61b2860b24dc208ca89a68d79dc00147125d531746d3ba808822ad" + "sha256:70e94a5f9bd46b26b63a41fb441ad35f2ae8862ad9d90765b6fa31ccc02c0a19", + "sha256:8d161a97a25eb381721b4b7251d5126ef4ec57e452114250b3e51ba5e4ff45a4" ], "markers": "python_version >= '3.7'", - "version": "==1.31.76" + "version": "==1.31.66" }, "cachetools": { "hashes": [ - "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2", - "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1" + "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590", + "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==5.3.2" + "version": "==5.3.1" }, "certifi": { "hashes": [ @@ -132,128 +132,128 @@ }, "charset-normalizer": { "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", + "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786", + "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e", + "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8", + "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", + "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa", + "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d", + "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", + "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", + "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895", + "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d", + "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a", + "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382", + "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678", + "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b", + "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e", + "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741", + "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4", + "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596", + "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9", + "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69", + "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c", + "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77", + "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13", + "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459", + "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e", + "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7", + "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908", + "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a", + "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f", + "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8", + "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482", + "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d", + "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d", + "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545", + "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34", + "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", + "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", + "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe", + "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e", + "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc", + "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7", + "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", + "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c", + "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557", + "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", + "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89", + "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078", + "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e", + "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4", + "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403", + "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0", + "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", + "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", + "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9", + "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05", + "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", + "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec", + "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56", + "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38", + "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", + "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c", + "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e", + "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd", + "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186", + "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455", + "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c", + "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65", + "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78", + "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287", + "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df", + "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", + "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1", + "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7", + "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989", + "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a", + "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63", + "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884", + "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649", + "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810", + "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", + "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4", + "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", + "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd", + "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5", + "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe", + "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293", + "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e", + "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", + "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" + "version": "==3.3.0" }, "cryptography": { "hashes": [ - "sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf", - "sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84", - "sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e", - "sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8", - "sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7", - "sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1", - "sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88", - "sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86", - "sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179", - "sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81", - "sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20", - "sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548", - "sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d", - "sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d", - "sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5", - "sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1", - "sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147", - "sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936", - "sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797", - "sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696", - "sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72", - "sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da", - "sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723" + "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67", + "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311", + "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8", + "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13", + "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143", + "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f", + "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829", + "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd", + "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397", + "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac", + "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d", + "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a", + "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839", + "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e", + "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6", + "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9", + "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860", + "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca", + "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91", + "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d", + "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714", + "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb", + "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f" ], "markers": "python_version >= '3.7'", - "version": "==41.0.5" + "version": "==41.0.4" }, "defusedxml": { "hashes": [ @@ -279,12 +279,12 @@ }, "django": { "hashes": [ - "sha256:8e0f1c2c2786b5c0e39fe1afce24c926040fad47c8ea8ad30aaf1188df29fc41", - "sha256:e1d37c51ad26186de355cbcec16613ebdabfa9689bbade9c538835205a8abbe9" + "sha256:08f41f468b63335aea0d904c5729e0250300f6a1907bf293a65499496cdbc68f", + "sha256:a64d2487cdb00ad7461434320ccc38e60af9c404773a2f95ab0093b4453a3215" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.7" + "version": "==4.2.6" }, "django-allow-cidr": { "hashes": [ @@ -310,15 +310,6 @@ ], "version": "==3.4.4" }, - "django-cors-headers": { - "hashes": [ - "sha256:25aabc94d4837678c1edf442c7f68a5f5fd151f6767b0e0b01c61a2179d02711", - "sha256:bd36c7aea0d070e462f3383f0dc9ef717e5fdc2b10a99c98c285f16da84ffba2" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==4.3.0" - }, "django-csp": { "hashes": [ "sha256:01443a07723f9a479d498bd7bb63571aaa771e690f64bde515db6cdb76e8041a", @@ -375,11 +366,11 @@ }, "faker": { "hashes": [ - "sha256:14ccb0aec342d33aa3889a864a56e5b3c2d56bce1b89f9189f4fbc128b9afc1e", - "sha256:da880a76322db7a879c848a0771e129338e0a680a9f695fd9a3e7a6ac82b45e1" + "sha256:a62a3fd3bfa3122d4f57dfa26a1cc37d76751a76c8ddd63cf9d24078c57913a4", + "sha256:e28090068293c5a83e7f4d636417d45fae1031ca8a8136cc2415549ebc2111e2" ], "markers": "python_version >= '3.8'", - "version": "==19.13.0" + "version": "==19.11.0" }, "fred-epplib": { "git": "https://github.com/cisagov/epplib.git", @@ -448,71 +439,76 @@ }, "geventconnpool": { "git": "https://github.com/rasky/geventconnpool.git", - "ref": null + "ref": "1bbb93a714a331a069adf27265fe582d9ba7ecd4" }, "greenlet": { "hashes": [ - "sha256:0a02d259510b3630f330c86557331a3b0e0c79dac3d166e449a39363beaae174", - "sha256:0b6f9f8ca7093fd4433472fd99b5650f8a26dcd8ba410e14094c1e44cd3ceddd", - "sha256:100f78a29707ca1525ea47388cec8a049405147719f47ebf3895e7509c6446aa", - "sha256:1757936efea16e3f03db20efd0cd50a1c86b06734f9f7338a90c4ba85ec2ad5a", - "sha256:19075157a10055759066854a973b3d1325d964d498a805bb68a1f9af4aaef8ec", - "sha256:19bbdf1cce0346ef7341705d71e2ecf6f41a35c311137f29b8a2dc2341374565", - "sha256:20107edf7c2c3644c67c12205dc60b1bb11d26b2610b276f97d666110d1b511d", - "sha256:22f79120a24aeeae2b4471c711dcf4f8c736a2bb2fabad2a67ac9a55ea72523c", - "sha256:2847e5d7beedb8d614186962c3d774d40d3374d580d2cbdab7f184580a39d234", - "sha256:28e89e232c7593d33cac35425b58950789962011cc274aa43ef8865f2e11f46d", - "sha256:329c5a2e5a0ee942f2992c5e3ff40be03e75f745f48847f118a3cfece7a28546", - "sha256:337322096d92808f76ad26061a8f5fccb22b0809bea39212cd6c406f6a7060d2", - "sha256:3fcc780ae8edbb1d050d920ab44790201f027d59fdbd21362340a85c79066a74", - "sha256:41bdeeb552d814bcd7fb52172b304898a35818107cc8778b5101423c9017b3de", - "sha256:4eddd98afc726f8aee1948858aed9e6feeb1758889dfd869072d4465973f6bfd", - "sha256:52e93b28db27ae7d208748f45d2db8a7b6a380e0d703f099c949d0f0d80b70e9", - "sha256:55d62807f1c5a1682075c62436702aaba941daa316e9161e4b6ccebbbf38bda3", - "sha256:5805e71e5b570d490938d55552f5a9e10f477c19400c38bf1d5190d760691846", - "sha256:599daf06ea59bfedbec564b1692b0166a0045f32b6f0933b0dd4df59a854caf2", - "sha256:60d5772e8195f4e9ebf74046a9121bbb90090f6550f81d8956a05387ba139353", - "sha256:696d8e7d82398e810f2b3622b24e87906763b6ebfd90e361e88eb85b0e554dc8", - "sha256:6e6061bf1e9565c29002e3c601cf68569c450be7fc3f7336671af7ddb4657166", - "sha256:80ac992f25d10aaebe1ee15df45ca0d7571d0f70b645c08ec68733fb7a020206", - "sha256:816bd9488a94cba78d93e1abb58000e8266fa9cc2aa9ccdd6eb0696acb24005b", - "sha256:85d2b77e7c9382f004b41d9c72c85537fac834fb141b0296942d52bf03fe4a3d", - "sha256:87c8ceb0cf8a5a51b8008b643844b7f4a8264a2c13fcbcd8a8316161725383fe", - "sha256:89ee2e967bd7ff85d84a2de09df10e021c9b38c7d91dead95b406ed6350c6997", - "sha256:8bef097455dea90ffe855286926ae02d8faa335ed8e4067326257cb571fc1445", - "sha256:8d11ebbd679e927593978aa44c10fc2092bc454b7d13fdc958d3e9d508aba7d0", - "sha256:91e6c7db42638dc45cf2e13c73be16bf83179f7859b07cfc139518941320be96", - "sha256:97e7ac860d64e2dcba5c5944cfc8fa9ea185cd84061c623536154d5a89237884", - "sha256:990066bff27c4fcf3b69382b86f4c99b3652bab2a7e685d968cd4d0cfc6f67c6", - "sha256:9fbc5b8f3dfe24784cee8ce0be3da2d8a79e46a276593db6868382d9c50d97b1", - "sha256:ac4a39d1abae48184d420aa8e5e63efd1b75c8444dd95daa3e03f6c6310e9619", - "sha256:b2c02d2ad98116e914d4f3155ffc905fd0c025d901ead3f6ed07385e19122c94", - "sha256:b2d3337dcfaa99698aa2377c81c9ca72fcd89c07e7eb62ece3f23a3fe89b2ce4", - "sha256:b489c36d1327868d207002391f662a1d163bdc8daf10ab2e5f6e41b9b96de3b1", - "sha256:b641161c302efbb860ae6b081f406839a8b7d5573f20a455539823802c655f63", - "sha256:b8ba29306c5de7717b5761b9ea74f9c72b9e2b834e24aa984da99cbfc70157fd", - "sha256:b9934adbd0f6e476f0ecff3c94626529f344f57b38c9a541f87098710b18af0a", - "sha256:ce85c43ae54845272f6f9cd8320d034d7a946e9773c693b27d620edec825e376", - "sha256:cf868e08690cb89360eebc73ba4be7fb461cfbc6168dd88e2fbbe6f31812cd57", - "sha256:d2905ce1df400360463c772b55d8e2518d0e488a87cdea13dd2c71dcb2a1fa16", - "sha256:d57e20ba591727da0c230ab2c3f200ac9d6d333860d85348816e1dca4cc4792e", - "sha256:d6a8c9d4f8692917a3dc7eb25a6fb337bff86909febe2f793ec1928cd97bedfc", - "sha256:d923ff276f1c1f9680d32832f8d6c040fe9306cbfb5d161b0911e9634be9ef0a", - "sha256:daa7197b43c707462f06d2c693ffdbb5991cbb8b80b5b984007de431493a319c", - "sha256:dbd4c177afb8a8d9ba348d925b0b67246147af806f0b104af4d24f144d461cd5", - "sha256:dc4d815b794fd8868c4d67602692c21bf5293a75e4b607bb92a11e821e2b859a", - "sha256:e9d21aaa84557d64209af04ff48e0ad5e28c5cca67ce43444e939579d085da72", - "sha256:ea6b8aa9e08eea388c5f7a276fabb1d4b6b9d6e4ceb12cc477c3d352001768a9", - "sha256:eabe7090db68c981fca689299c2d116400b553f4b713266b130cfc9e2aa9c5a9", - "sha256:f2f6d303f3dee132b322a14cd8765287b8f86cdc10d2cb6a6fae234ea488888e", - "sha256:f33f3258aae89da191c6ebaa3bc517c6c4cbc9b9f689e5d8452f7aedbb913fa8", - "sha256:f7bfb769f7efa0eefcd039dd19d843a4fbfbac52f1878b1da2ed5793ec9b1a65", - "sha256:f89e21afe925fcfa655965ca8ea10f24773a1791400989ff32f467badfe4a064", - "sha256:fa24255ae3c0ab67e613556375a4341af04a084bd58764731972bcbc8baeba36" + "sha256:02a807b2a58d5cdebb07050efe3d7deaf915468d112dfcf5e426d0564aa3aa4a", + "sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c", + "sha256:0d3f83ffb18dc57243e0151331e3c383b05e5b6c5029ac29f754745c800f8ed9", + "sha256:10b5582744abd9858947d163843d323d0b67be9432db50f8bf83031032bc218d", + "sha256:123910c58234a8d40eaab595bc56a5ae49bdd90122dde5bdc012c20595a94c14", + "sha256:1482fba7fbed96ea7842b5a7fc11d61727e8be75a077e603e8ab49d24e234383", + "sha256:19834e3f91f485442adc1ee440171ec5d9a4840a1f7bd5ed97833544719ce10b", + "sha256:1d363666acc21d2c204dd8705c0e0457d7b2ee7a76cb16ffc099d6799744ac99", + "sha256:211ef8d174601b80e01436f4e6905aca341b15a566f35a10dd8d1e93f5dbb3b7", + "sha256:269d06fa0f9624455ce08ae0179430eea61085e3cf6457f05982b37fd2cefe17", + "sha256:2e7dcdfad252f2ca83c685b0fa9fba00e4d8f243b73839229d56ee3d9d219314", + "sha256:334ef6ed8337bd0b58bb0ae4f7f2dcc84c9f116e474bb4ec250a8bb9bd797a66", + "sha256:343675e0da2f3c69d3fb1e894ba0a1acf58f481f3b9372ce1eb465ef93cf6fed", + "sha256:37f60b3a42d8b5499be910d1267b24355c495064f271cfe74bf28b17b099133c", + "sha256:38ad562a104cd41e9d4644f46ea37167b93190c6d5e4048fcc4b80d34ecb278f", + "sha256:3c0d36f5adc6e6100aedbc976d7428a9f7194ea79911aa4bf471f44ee13a9464", + "sha256:3fd2b18432e7298fcbec3d39e1a0aa91ae9ea1c93356ec089421fabc3651572b", + "sha256:4a1a6244ff96343e9994e37e5b4839f09a0207d35ef6134dce5c20d260d0302c", + "sha256:4cd83fb8d8e17633ad534d9ac93719ef8937568d730ef07ac3a98cb520fd93e4", + "sha256:527cd90ba3d8d7ae7dceb06fda619895768a46a1b4e423bdb24c1969823b8362", + "sha256:56867a3b3cf26dc8a0beecdb4459c59f4c47cdd5424618c08515f682e1d46692", + "sha256:621fcb346141ae08cb95424ebfc5b014361621b8132c48e538e34c3c93ac7365", + "sha256:63acdc34c9cde42a6534518e32ce55c30f932b473c62c235a466469a710bfbf9", + "sha256:6512592cc49b2c6d9b19fbaa0312124cd4c4c8a90d28473f86f92685cc5fef8e", + "sha256:6672fdde0fd1a60b44fb1751a7779c6db487e42b0cc65e7caa6aa686874e79fb", + "sha256:6a5b2d4cdaf1c71057ff823a19d850ed5c6c2d3686cb71f73ae4d6382aaa7a06", + "sha256:6a68d670c8f89ff65c82b936275369e532772eebc027c3be68c6b87ad05ca695", + "sha256:6bb36985f606a7c49916eff74ab99399cdfd09241c375d5a820bb855dfb4af9f", + "sha256:73b2f1922a39d5d59cc0e597987300df3396b148a9bd10b76a058a2f2772fc04", + "sha256:7709fd7bb02b31908dc8fd35bfd0a29fc24681d5cc9ac1d64ad07f8d2b7db62f", + "sha256:8060b32d8586e912a7b7dac2d15b28dbbd63a174ab32f5bc6d107a1c4143f40b", + "sha256:80dcd3c938cbcac986c5c92779db8e8ce51a89a849c135172c88ecbdc8c056b7", + "sha256:813720bd57e193391dfe26f4871186cf460848b83df7e23e6bef698a7624b4c9", + "sha256:831d6f35037cf18ca5e80a737a27d822d87cd922521d18ed3dbc8a6967be50ce", + "sha256:871b0a8835f9e9d461b7fdaa1b57e3492dd45398e87324c047469ce2fc9f516c", + "sha256:952256c2bc5b4ee8df8dfc54fc4de330970bf5d79253c863fb5e6761f00dda35", + "sha256:96d9ea57292f636ec851a9bb961a5cc0f9976900e16e5d5647f19aa36ba6366b", + "sha256:9a812224a5fb17a538207e8cf8e86f517df2080c8ee0f8c1ed2bdaccd18f38f4", + "sha256:9adbd8ecf097e34ada8efde9b6fec4dd2a903b1e98037adf72d12993a1c80b51", + "sha256:9de687479faec7db5b198cc365bc34addd256b0028956501f4d4d5e9ca2e240a", + "sha256:a048293392d4e058298710a54dfaefcefdf49d287cd33fb1f7d63d55426e4355", + "sha256:aa15a2ec737cb609ed48902b45c5e4ff6044feb5dcdfcf6fa8482379190330d7", + "sha256:abe1ef3d780de56defd0c77c5ba95e152f4e4c4e12d7e11dd8447d338b85a625", + "sha256:ad6fb737e46b8bd63156b8f59ba6cdef46fe2b7db0c5804388a2d0519b8ddb99", + "sha256:b1660a15a446206c8545edc292ab5c48b91ff732f91b3d3b30d9a915d5ec4779", + "sha256:b505fcfc26f4148551826a96f7317e02c400665fa0883fe505d4fcaab1dabfdd", + "sha256:b822fab253ac0f330ee807e7485769e3ac85d5eef827ca224feaaefa462dc0d0", + "sha256:bdd696947cd695924aecb3870660b7545a19851f93b9d327ef8236bfc49be705", + "sha256:bdfaeecf8cc705d35d8e6de324bf58427d7eafb55f67050d8f28053a3d57118c", + "sha256:be557119bf467d37a8099d91fbf11b2de5eb1fd5fc5b91598407574848dc910f", + "sha256:c6b5ce7f40f0e2f8b88c28e6691ca6806814157ff05e794cdd161be928550f4c", + "sha256:c94e4e924d09b5a3e37b853fe5924a95eac058cb6f6fb437ebb588b7eda79870", + "sha256:cc3e2679ea13b4de79bdc44b25a0c4fcd5e94e21b8f290791744ac42d34a0353", + "sha256:d1e22c22f7826096ad503e9bb681b05b8c1f5a8138469b255eb91f26a76634f2", + "sha256:d5539f6da3418c3dc002739cb2bb8d169056aa66e0c83f6bacae0cd3ac26b423", + "sha256:d55db1db455c59b46f794346efce896e754b8942817f46a1bada2d29446e305a", + "sha256:e09dea87cc91aea5500262993cbd484b41edf8af74f976719dd83fe724644cd6", + "sha256:e52a712c38e5fb4fd68e00dc3caf00b60cb65634d50e32281a9d6431b33b4af1", + "sha256:e693e759e172fa1c2c90d35dea4acbdd1d609b6936115d3739148d5e4cd11947", + "sha256:ecf94aa539e97a8411b5ea52fc6ccd8371be9550c4041011a091eb8b3ca1d810", + "sha256:f351479a6914fd81a55c8e68963609f792d9b067fb8a60a042c585a621e0de4f", + "sha256:f47932c434a3c8d3c86d865443fadc1fbf574e9b11d6650b656e602b1797908a" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==3.0.1" + "version": "==3.0.0" }, "gunicorn": { "hashes": [ @@ -745,10 +741,10 @@ }, "phonenumberslite": { "hashes": [ - "sha256:1e03f7076ab2f010088b1b8041ebdc42acd3b797e8f45997ab1861cdaea76851", - "sha256:adce353ee15b75f2deccf0eff77bada2a3d036f49ccfb30b8c172dd814fd51e9" + "sha256:7c719e35ef551a895459382e9faf592f52647312dd90b543b06460aa0e1c49c4", + "sha256:cf6cf56c889c6787ec6b30b5791693f6dd678f633358f4aeea1fddf98d4cadcb" ], - "version": "==8.13.24" + "version": "==8.13.23" }, "psycopg2-binary": { "hashes": [ @@ -779,7 +775,6 @@ "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291", "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe", "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980", - "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93", "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692", "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119", "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716", @@ -789,7 +784,6 @@ "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc", "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c", "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5", - "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab", "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984", "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9", "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf", @@ -802,7 +796,6 @@ "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041", "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7", "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860", - "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d", "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245", "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27", "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417", @@ -1172,45 +1165,45 @@ }, "black": { "hashes": [ - "sha256:037e9b4664cafda5f025a1728c50a9e9aedb99a759c89f760bd83730e76ba884", - "sha256:1b917a2aa020ca600483a7b340c165970b26e9029067f019e3755b56e8dd5916", - "sha256:1f8ce316753428ff68749c65a5f7844631aa18c8679dfd3ca9dc1a289979c258", - "sha256:33d40f5b06be80c1bbce17b173cda17994fbad096ce60eb22054da021bf933d1", - "sha256:3f157a8945a7b2d424da3335f7ace89c14a3b0625e6593d21139c2d8214d55ce", - "sha256:5ed45ac9a613fb52dad3b61c8dea2ec9510bf3108d4db88422bacc7d1ba1243d", - "sha256:6d23d7822140e3fef190734216cefb262521789367fbdc0b3f22af6744058982", - "sha256:7670242e90dc129c539e9ca17665e39a146a761e681805c54fbd86015c7c84f7", - "sha256:7b4d10b0f016616a0d93d24a448100adf1699712fb7a4efd0e2c32bbb219b173", - "sha256:7cb5936e686e782fddb1c73f8aa6f459e1ad38a6a7b0e54b403f1f05a1507ee9", - "sha256:7d56124b7a61d092cb52cce34182a5280e160e6aff3137172a68c2c2c4b76bcb", - "sha256:840015166dbdfbc47992871325799fd2dc0dcf9395e401ada6d88fe11498abad", - "sha256:9c74de4c77b849e6359c6f01987e94873c707098322b91490d24296f66d067dc", - "sha256:b15b75fc53a2fbcac8a87d3e20f69874d161beef13954747e053bca7a1ce53a0", - "sha256:cfcce6f0a384d0da692119f2d72d79ed07c7159879d0bb1bb32d2e443382bf3a", - "sha256:d431e6739f727bb2e0495df64a6c7a5310758e87505f5f8cde9ff6c0f2d7e4fe", - "sha256:e293e4c2f4a992b980032bbd62df07c1bcff82d6964d6c9496f2cd726e246ace", - "sha256:ec3f8e6234c4e46ff9e16d9ae96f4ef69fa328bb4ad08198c8cee45bb1f08c69" + "sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699", + "sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e", + "sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171", + "sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd", + "sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9", + "sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b", + "sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23", + "sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204", + "sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747", + "sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8", + "sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a", + "sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c", + "sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604", + "sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a", + "sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e", + "sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd", + "sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c", + "sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==23.10.1" + "version": "==23.10.0" }, "blinker": { "hashes": [ - "sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9", - "sha256:e6820ff6fa4e4d1d8e2747c2283749c3f547e4fee112b98555cdcdae32996182" + "sha256:152090d27c1c5c722ee7e48504b02d76502811ce02e1523553b4cf8c8b3d3a8d", + "sha256:296320d6c28b006eb5e32d4712202dbcdcbf5dc482da298c2f44881c43884aaa" ], - "markers": "python_version >= '3.8'", - "version": "==1.7.0" + "markers": "python_version >= '3.7'", + "version": "==1.6.3" }, "boto3": { "hashes": [ - "sha256:85e2fa361ad3210d30800bad311688261f2673a9b301e0edab56463d89609761", - "sha256:d18688bc5d688decf3cc404430a3ac3ec317be653cdcfbc51104c01f38a66434" + "sha256:38658585791f47cca3fc6aad03838de0136778b533e8c71c6a9590aedc60fbde", + "sha256:a8228522c7db33694c0746dec8b48c05473671626359dd62ab6829eb7871eddc" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.28.76" + "version": "==1.28.66" }, "boto3-mocking": { "hashes": [ @@ -1223,28 +1216,28 @@ }, "boto3-stubs": { "hashes": [ - "sha256:d89c3546e9e500f81ebfe78c71627e74085d3f77cd7e62830b5e48a67bce9b75", - "sha256:fc57fc32d9a0c4bdd02676c37dbaa911b3e6c3857e417a229d236938d31299fe" + "sha256:06e696ce8529f899a2ba388d6604ca8ed8ba367dd53898e27b4ce49e8b3fd2aa", + "sha256:b85689c50a6768bb0fcb85e06394d7898b330b82f34cec26c36d912e6a41280d" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==1.28.76" + "version": "==1.28.66" }, "botocore": { "hashes": [ - "sha256:479abb5a1ee03eb00faa1ea176bc595b2f46f7494777807681a9df45ed99ea18", - "sha256:74e0a4515d61b2860b24dc208ca89a68d79dc00147125d531746d3ba808822ad" + "sha256:70e94a5f9bd46b26b63a41fb441ad35f2ae8862ad9d90765b6fa31ccc02c0a19", + "sha256:8d161a97a25eb381721b4b7251d5126ef4ec57e452114250b3e51ba5e4ff45a4" ], "markers": "python_version >= '3.7'", - "version": "==1.31.76" + "version": "==1.31.66" }, "botocore-stubs": { "hashes": [ - "sha256:778c6e014ae1103d546d59dffb000b0a659c9b9bbfa11050ff4a62c5adeec3a4", - "sha256:9fd9447a28642efa35a1c5590fc35132cf0173cd12055ba9044511cb6b24dd6f" + "sha256:5ea5f4af18ee654cf510d69b3bc7c1ed3236b50fcd4e3eb98c11d28033ff05c3", + "sha256:b65fa3ff36e8a70518a143b5025559918a68d7a20b85c88f8a1f067f6620a205" ], "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==1.31.76" + "version": "==1.31.66" }, "click": { "hashes": [ @@ -1256,12 +1249,12 @@ }, "django": { "hashes": [ - "sha256:8e0f1c2c2786b5c0e39fe1afce24c926040fad47c8ea8ad30aaf1188df29fc41", - "sha256:e1d37c51ad26186de355cbcec16613ebdabfa9689bbade9c538835205a8abbe9" + "sha256:08f41f468b63335aea0d904c5729e0250300f6a1907bf293a65499496cdbc68f", + "sha256:a64d2487cdb00ad7461434320ccc38e60af9c404773a2f95ab0093b4453a3215" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.7" + "version": "==4.2.6" }, "django-debug-toolbar": { "hashes": [ @@ -1281,12 +1274,12 @@ }, "django-stubs": { "hashes": [ - "sha256:2fcd257884a68dfa02de41ee5410ec805264d9b07d9b5b119e4dea82c7b8345e", - "sha256:e60b43de662a199db4b15c803c06669e0ac5035614af291cbd3b91591f7dcc94" + "sha256:5a23cf622f1426a0b0c48bd6e2ef709a66275d72073baf6fdf5ac36fc4cce736", + "sha256:706b2456bd0e56c468dfd8f27b0e7dde001c5c7cd3010d67fcbda9d95467e050" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.6" + "version": "==4.2.5" }, "django-stubs-ext": { "hashes": [ @@ -1315,11 +1308,11 @@ }, "gitdb": { "hashes": [ - "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4", - "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b" + "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a", + "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7" ], "markers": "python_version >= '3.7'", - "version": "==4.0.11" + "version": "==4.0.10" }, "gitpython": { "hashes": [ @@ -1597,20 +1590,19 @@ }, "types-awscrt": { "hashes": [ - "sha256:4eb4f3bd0c41a2710cacda13098374da9faa76c5a0fb901aa5659e0fd48ceda1", - "sha256:a2d534b7017c3476ee69a44bd8aeaf3b588c42baa8322473d100a45ee67510d7" + "sha256:7b55f5a12ccd4407bc8f1e35c69bb40c931f8513ce1ad81a4527fce3989003fd", + "sha256:9a21caac4287c113dd52665707785c45bb1d3242b7a2b8aeb57c49e9e749a330" ], "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==0.19.8" + "version": "==0.19.3" }, "types-cachetools": { "hashes": [ - "sha256:27c982cdb9cf3fead8b0089ee6b895715ecc99dac90ec29e2cab56eb1aaf4199", - "sha256:98c069dc7fc087b1b061703369c80751b0a0fc561f6fb072b554e5eee23773a0" + "sha256:595f0342d246c8ba534f5a762cf4c2f60ecb61e8002b8b2277fd5cf791d4e851", + "sha256:f7f8a25bfe306f2e6bc2ad0a2f949d9e72f2d91036d509c36d3810bf728bc6e1" ], "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==5.3.0.7" + "version": "==5.3.0.6" }, "types-pytz": { "hashes": [ diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 080a343ee..68ff51597 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -187,7 +187,7 @@ h1, h2, h3 { .object-tools li a, .object-tools p a { font-family: "Source Sans Pro Web", "Helvetica Neue", Helvetica, Roboto, Arial, sans-serif; - text-transform: none!important; + text-transform: capitalize!important; font-size: 14px!important; } diff --git a/src/registrar/assets/sass/_theme/_alerts.scss b/src/registrar/assets/sass/_theme/_alerts.scss index 9ee28a357..dd51734ed 100644 --- a/src/registrar/assets/sass/_theme/_alerts.scss +++ b/src/registrar/assets/sass/_theme/_alerts.scss @@ -5,11 +5,7 @@ font-size: units(3); } } - -.usa-alert__text.measure-none { - max-width: measure(none); -} - + // The icon was off center for some reason // Fixes that issue @media (min-width: 64em){ diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 896691efb..e0dedb60c 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -119,8 +119,6 @@ INSTALLED_APPS = [ "api", # Only for generating documentation, uncomment to run manage.py generate_puml # "puml_generator", - # supports necessary headers for Django cross origin - "corsheaders", ] # Middleware are routines for processing web requests. @@ -129,8 +127,6 @@ INSTALLED_APPS = [ MIDDLEWARE = [ # django-allow-cidr: enable use of CIDR IP ranges in ALLOWED_HOSTS "allow_cidr.middleware.AllowCIDRMiddleware", - # django-cors-headers: listen to cors responses - "corsheaders.middleware.CorsMiddleware", # serve static assets in production "whitenoise.middleware.WhiteNoiseMiddleware", # provide security enhancements to the request/response cycle @@ -296,11 +292,6 @@ CSP_DEFAULT_SRC = allowed_sources CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources -# Cross-Origin Resource Sharing (CORS) configuration -# Sets clients that allow access control to manage.get.gov -# TODO: remove :8080 to see if we can have all localhost access -CORS_ALLOWED_ORIGINS = ["http://localhost:8080", "https://beta.get.gov"] - # Content-Length header is set by django.middleware.common.CommonMiddleware diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 1e595e042..ea0e827fa 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -38,8 +38,8 @@ class Command(BaseCommand): """ OPTIONAL ARGUMENTS: --runMigrations - Triggers running all scripts (in sequence) - for transition domain migrations + A boolean (default to true), which triggers running + all scripts (in sequence) for transition domain migrations --migrationDirectory The location of the files used for load_transition_domain migration script @@ -61,11 +61,11 @@ class Command(BaseCommand): (usually this can remain at default value of |) --debug - Activates additional print statements + A boolean (default to true), which activates additional print statements --prompt - Activates terminal prompts that allows - the user to step through each portion of this + A boolean (default to true), which activates terminal prompts + that allows the user to step through each portion of this script. --limitParse diff --git a/src/registrar/templates/base.html b/src/registrar/templates/base.html index 68a5c69ea..72c30f323 100644 --- a/src/registrar/templates/base.html +++ b/src/registrar/templates/base.html @@ -62,11 +62,10 @@ {% if IS_DEMO_SITE %}
-
+
-

New domain requests are paused

-

- This is the new registrar for managing .gov domains. Note that we’re not accepting requests for new .gov domains until January 2024. Follow .gov updates at get.gov/updates/. +

+ BETA SITE: We’re building a new way to get a .gov. Take a look around, but don’t rely on this site yet. This site is for testing purposes only. Don’t enter real data into any form on this site. To learn about requesting a .gov domain, visit get.gov

diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index 22df74685..68fdbe7aa 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -15,7 +15,7 @@ {% if has_add_permission %}
  • - Add domain + Add Domain
    • {% endif %} diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index d9127d72c..ffada0a0b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,8 +2,6 @@ import csv from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact -from django.db.models import Value -from django.db.models.functions import Coalesce def export_domains_to_writer(writer, columns, sort_fields, filter_condition): @@ -63,13 +61,7 @@ def export_data_type_to_csv(csv_file): "Status", "Expiration Date", ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] + sort_fields = ["domain__name"] filter_condition = { "domain__state__in": [ Domain.State.READY, @@ -92,13 +84,7 @@ def export_data_full_to_csv(csv_file): "State", "Security Contact Email", ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] + sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { "domain__state__in": [ Domain.State.READY, @@ -121,13 +107,7 @@ def export_data_federal_to_csv(csv_file): "State", "Security Contact Email", ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] + sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { "organization_type__icontains": "federal", "domain__state__in": [ diff --git a/src/requirements.txt b/src/requirements.txt index 125bd3d2f..ff289ea63 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,35 +1,34 @@ -i https://pypi.python.org/simple annotated-types==0.6.0; python_version >= '3.8' asgiref==3.7.2; python_version >= '3.7' -boto3==1.28.76; python_version >= '3.7' -botocore==1.31.76; python_version >= '3.7' -cachetools==5.3.2; python_version >= '3.7' +boto3==1.28.66; python_version >= '3.7' +botocore==1.31.66; python_version >= '3.7' +cachetools==5.3.1; python_version >= '3.7' certifi==2023.7.22; python_version >= '3.6' cfenv==0.5.3 cffi==1.16.0; python_version >= '3.8' -charset-normalizer==3.3.2; python_full_version >= '3.7.0' -cryptography==41.0.5; python_version >= '3.7' +charset-normalizer==3.3.0; python_full_version >= '3.7.0' +cryptography==41.0.4; python_version >= '3.7' defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' dj-database-url==2.1.0 dj-email-url==1.0.6 -django==4.2.7; python_version >= '3.8' +django==4.2.6; python_version >= '3.8' django-allow-cidr==0.7.1 django-auditlog==2.3.0; python_version >= '3.7' django-cache-url==3.4.4 -django-cors-headers==4.3.0; python_version >= '3.8' django-csp==3.7 django-fsm==2.8.1 django-login-required-middleware==0.9.0 django-phonenumber-field[phonenumberslite]==7.2.0; python_version >= '3.8' django-widget-tweaks==1.5.0; python_version >= '3.8' environs[django]==9.5.0; python_version >= '3.6' -faker==19.13.0; python_version >= '3.8' +faker==19.11.0; python_version >= '3.8' fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a3a9a345f0ef561c furl==2.1.3 future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' gevent==23.9.1; python_version >= '3.8' -geventconnpool@ git+https://github.com/rasky/geventconnpool.git -greenlet==3.0.1; python_version >= '3.7' +geventconnpool@ git+https://github.com/rasky/geventconnpool.git@1bbb93a714a331a069adf27265fe582d9ba7ecd4 +greenlet==3.0.0; python_version >= '3.7' gunicorn==21.2.0; python_version >= '3.5' idna==3.4; python_version >= '3.5' jmespath==1.0.1; python_version >= '3.7' @@ -40,7 +39,7 @@ marshmallow==3.20.1; python_version >= '3.8' oic==1.6.1; python_version ~= '3.7' orderedmultidict==1.0.1 packaging==23.2; python_version >= '3.7' -phonenumberslite==8.13.24 +phonenumberslite==8.13.23 psycopg2-binary==2.9.9; python_version >= '3.7' pycparser==2.21 pycryptodomex==3.19.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' From a8188cb08f05e031ffc638f7946b933eeaa850cf Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 15:02:06 -0600 Subject: [PATCH 45/88] Started updates to master migration file. Made update to terminal_helper so prompt no longer provides "skip" option --- .../commands/master_domain_migrations.py | 15 +++++++++++++++ .../commands/utility/terminal_helper.py | 11 +++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index ea0e827fa..22f8a004b 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -95,6 +95,19 @@ class Command(BaseCommand): ) # The following file arguments have default values for running in the sandbox + + # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) + # TODO: use the migration directory arg or force user to type FULL filepath? + parser.add_argument( + "--migrationJson", + default="/app/management/commands/utility/dataFile.json", # TODO: Get rid of this once done? Or leave it as defaults?? + help=( + "A JSON file that holds the location and filenames" + "of all the data files used for migrations" + ), + ) + + # TODO: deprecate this once JSON module is done? (or keep as an override) parser.add_argument( "--migrationDirectory", default="migrationdata", @@ -103,6 +116,8 @@ class Command(BaseCommand): "load_transition_domain migration script" ), ) + + # TODO: deprecate this once JSON module is done? (or keep as an override) parser.add_argument( "--migrationFilenames", default="escrow_domain_contacts.daily.gov.GOV.txt," diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 5ec9408cd..4f937e699 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -81,7 +81,7 @@ class TerminalHelper: The "answer" return value is True for "yes" or False for "no". """ - valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit", "s": "skip"} + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit"} if default is None: prompt = " [y/n] " elif default == "yes": @@ -148,12 +148,13 @@ class TerminalHelper: """Create to reduce code complexity. Prompts the user to inspect the given string and asks if they wish to proceed. - Returns true if the user responds (y), - Returns false if the user responds (n)""" + If the user responds (y), returns TRUE + If the user responds (n), either returns FALSE + or exits the system if system_exit_on_terminate = TRUE""" action_description_for_selecting_no = "skip, E = exit" if system_exit_on_terminate: - action_description_for_selecting_no = "exit, S = skip" + action_description_for_selecting_no = "exit" # Allow the user to inspect the command string # and ask if they wish to proceed @@ -176,8 +177,6 @@ class TerminalHelper: if system_exit_on_terminate: sys.exit() return False - if proceed_execution == "skip": - return False return True @staticmethod From bf374725cbf0776bc108575fdb633ac267329f62 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 15:11:38 -0600 Subject: [PATCH 46/88] finished adding "migrationJson" arg to master migration script --- .../management/commands/master_domain_migrations.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 22f8a004b..3019ac8ef 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -99,8 +99,7 @@ class Command(BaseCommand): # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( - "--migrationJson", - default="/app/management/commands/utility/dataFile.json", # TODO: Get rid of this once done? Or leave it as defaults?? + "migrationJson", help=( "A JSON file that holds the location and filenames" "of all the data files used for migrations" @@ -282,6 +281,7 @@ class Command(BaseCommand): # ====================================================== def run_load_transition_domain_script( self, + migrationJson: str, file_location: str, domain_contacts_filename: str, contacts_filename: str, @@ -372,6 +372,7 @@ class Command(BaseCommand): def run_migration_scripts( self, + migrationJson, file_location, domain_contacts_filename, contacts_filename, @@ -423,6 +424,7 @@ class Command(BaseCommand): # Proceed executing the migration scripts self.run_load_transition_domain_script( + migrationJson, file_location, domain_contacts_filename, contacts_filename, @@ -437,6 +439,7 @@ class Command(BaseCommand): def handle( self, + migrationJson, **options, ): """ @@ -531,6 +534,7 @@ class Command(BaseCommand): # Run migration scripts self.run_migration_scripts( + migrationJson, file_location, domain_contacts_filename, contacts_filename, From cb9b135178306af629791e7d422c0d32b8a023e9 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 15:19:23 -0600 Subject: [PATCH 47/88] Updated hand-off for JSON parsing --- .../commands/load_transition_domain.py | 36 +++++++++++++------ .../commands/master_domain_migrations.py | 20 +++++------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index d08b1c2ed..9496789f1 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -41,14 +41,11 @@ class Command(BaseCommand): for testing purposes, but USE WITH CAUTION """ parser.add_argument( - "domain_contacts_filename", help="Data file with domain contact information" - ) - parser.add_argument( - "contacts_filename", - help="Data file with contact information", - ) - parser.add_argument( - "domain_statuses_filename", help="Data file with domain status information" + "migration_Json_filename", + help=( + "A JSON file that holds the location and filenames" + "of all the data files used for migrations" + ), ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -71,6 +68,19 @@ class Command(BaseCommand): parser.add_argument( "--directory", default="migrationdata", help="Desired directory" ) + + parser.add_argument( + "--domain_contacts_filename", + help="Data file with domain contact information" + ) + parser.add_argument( + "--contacts_filename", + help="Data file with contact information", + ) + parser.add_argument( + "--domain_statuses_filename", + help="Data file with domain status information" + ) parser.add_argument( "--infer_filenames", action=argparse.BooleanOptionalAction, @@ -309,9 +319,7 @@ class Command(BaseCommand): def handle( # noqa: C901 self, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, + migration_Json_filename, **options, ): """Parse the data files and create TransitionDomains.""" @@ -336,6 +344,12 @@ class Command(BaseCommand): # (In the event they are stored seperately) directory = options.get("directory") + # Main script filenames + # TODO: @ZANDER to replace this with new TransitionDomainArgument object + domain_contacts_filename = directory+ options.get("domain_contacts_filename") + contacts_filename = directory+ options.get("contacts_filename") + domain_statuses_filename = directory+ options.get("domain_statuses_filename") + # Agency information agency_adhoc_filename = options.get("agency_adhoc_filename") # Federal agency / organization type information diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 3019ac8ef..673345b8a 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -99,7 +99,7 @@ class Command(BaseCommand): # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( - "migrationJson", + "migration_Json_filename", help=( "A JSON file that holds the location and filenames" "of all the data files used for migrations" @@ -281,7 +281,7 @@ class Command(BaseCommand): # ====================================================== def run_load_transition_domain_script( self, - migrationJson: str, + migration_Json_filename: str, file_location: str, domain_contacts_filename: str, contacts_filename: str, @@ -297,9 +297,7 @@ class Command(BaseCommand): command_script = "load_transition_domain" command_string = ( f"./manage.py {command_script} " - f"{file_location+domain_contacts_filename} " - f"{file_location+contacts_filename} " - f"{file_location+domain_statuses_filename} " + f"{file_location+migration_Json_filename}" ) if sep is not None and sep != "|": command_string += f"--sep {sep} " @@ -323,9 +321,7 @@ class Command(BaseCommand): if proceed: call_command( command_script, - f"{file_location+domain_contacts_filename}", - f"{file_location+contacts_filename}", - f"{file_location+domain_statuses_filename}", + f"{file_location+migration_Json_filename}", sep=sep, resetTable=reset_table, debug=debug_on, @@ -372,7 +368,7 @@ class Command(BaseCommand): def run_migration_scripts( self, - migrationJson, + migration_Json_filename, file_location, domain_contacts_filename, contacts_filename, @@ -424,7 +420,7 @@ class Command(BaseCommand): # Proceed executing the migration scripts self.run_load_transition_domain_script( - migrationJson, + migration_Json_filename, file_location, domain_contacts_filename, contacts_filename, @@ -439,7 +435,7 @@ class Command(BaseCommand): def handle( self, - migrationJson, + migration_Json_filename, **options, ): """ @@ -534,7 +530,7 @@ class Command(BaseCommand): # Run migration scripts self.run_migration_scripts( - migrationJson, + migration_Json_filename, file_location, domain_contacts_filename, contacts_filename, From eca8852d009ced42888a5e83111b3958cc2467ad Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:58:24 -0700 Subject: [PATCH 48/88] Parse json file --- .../commands/load_transition_domain.py | 33 ++++++++++++++++--- .../commands/master_domain_migrations.py | 22 +++++-------- .../utility/transition_domain_arguments.py | 1 + 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 9496789f1..7edf22457 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -1,3 +1,4 @@ +import json import sys import csv import logging @@ -41,7 +42,7 @@ class Command(BaseCommand): for testing purposes, but USE WITH CAUTION """ parser.add_argument( - "migration_Json_filename", + "migration_json_filename", help=( "A JSON file that holds the location and filenames" "of all the data files used for migrations" @@ -319,10 +320,32 @@ class Command(BaseCommand): def handle( # noqa: C901 self, - migration_Json_filename, + migration_json_filename, **options, ): """Parse the data files and create TransitionDomains.""" + ### Process JSON file ### + # If a JSON was provided, use its values instead of defaults. + # TODO: there is no way to discern user overrides from those arg’s defaults. + with open(migration_json_filename, "r") as jsonFile: + # load JSON object as a dictionary + try: + data = json.load(jsonFile) + # Create an instance of TransitionDomainArguments + args = TransitionDomainArguments() + # Iterate over the data from the JSON file + for key, value in data.items(): + # Check if the key exists in TransitionDomainArguments + if hasattr(args, key): + # If it does, update the options + options[key] = value + except Exception as err: + logger.error(f"""{TerminalColors.FAIL}There was an error loading the JSON responsible + for providing filepaths. + {TerminalColors.ENDC} + """) + raise err + sep = options.get("sep") # If --resetTable was used, prompt user to confirm @@ -346,9 +369,9 @@ class Command(BaseCommand): # Main script filenames # TODO: @ZANDER to replace this with new TransitionDomainArgument object - domain_contacts_filename = directory+ options.get("domain_contacts_filename") - contacts_filename = directory+ options.get("contacts_filename") - domain_statuses_filename = directory+ options.get("domain_statuses_filename") + domain_contacts_filename = directory + "/" + options.get("domain_contacts_filename") + contacts_filename = directory + "/" + options.get("contacts_filename") + domain_statuses_filename = directory + "/" + options.get("domain_statuses_filename") # Agency information agency_adhoc_filename = options.get("agency_adhoc_filename") diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 673345b8a..58124157f 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -99,7 +99,7 @@ class Command(BaseCommand): # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( - "migration_Json_filename", + "migration_json_filename", help=( "A JSON file that holds the location and filenames" "of all the data files used for migrations" @@ -281,11 +281,8 @@ class Command(BaseCommand): # ====================================================== def run_load_transition_domain_script( self, - migration_Json_filename: str, + migration_json_filename: str, file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, sep: str, reset_table: bool, debug_on: bool, @@ -297,7 +294,7 @@ class Command(BaseCommand): command_script = "load_transition_domain" command_string = ( f"./manage.py {command_script} " - f"{file_location+migration_Json_filename}" + f"{file_location+migration_json_filename}" ) if sep is not None and sep != "|": command_string += f"--sep {sep} " @@ -321,7 +318,7 @@ class Command(BaseCommand): if proceed: call_command( command_script, - f"{file_location+migration_Json_filename}", + f"{file_location+migration_json_filename}", sep=sep, resetTable=reset_table, debug=debug_on, @@ -368,7 +365,7 @@ class Command(BaseCommand): def run_migration_scripts( self, - migration_Json_filename, + migration_json_filename, file_location, domain_contacts_filename, contacts_filename, @@ -420,11 +417,8 @@ class Command(BaseCommand): # Proceed executing the migration scripts self.run_load_transition_domain_script( - migration_Json_filename, + migration_json_filename, file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, sep, reset_table, debug_on, @@ -435,7 +429,7 @@ class Command(BaseCommand): def handle( self, - migration_Json_filename, + migration_json_filename, **options, ): """ @@ -530,7 +524,7 @@ class Command(BaseCommand): # Run migration scripts self.run_migration_scripts( - migration_Json_filename, + migration_json_filename, file_location, domain_contacts_filename, contacts_filename, diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 70f13cfb5..335f74cd9 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -46,6 +46,7 @@ class TransitionDomainArguments: domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) domain_contacts_filename: Optional[str] = field(default=None, repr=True) domain_statuses_filename: Optional[str] = field(default=None, repr=True) + contacts_filename: Optional[str] = field(default=None, repr=True) # Flags # debug: Optional[bool] = field(default=False, repr=True) From e4b90e7dfe1803d35adc0f72b573a239ff1d7f80 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 15:59:11 -0600 Subject: [PATCH 49/88] possible fix for blanks around datafile delimiters --- .../utility/extra_transition_domain_helper.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7022944a0..a25730b2f 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -895,14 +895,16 @@ class ExtraTransitionDomain: def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: - reader = csv.DictReader(requested_file, delimiter=seperator) + reader = csv.DictReader(requested_file, skipinitialspace=True, delimiter=seperator) dict_data = {} for row in reader: + # cleaned_row, cleaning_was_needed = self.clean_delimiters() if None in row: - print("Skipping row with None key") - print(dataclass_type) + logger.info("Skipping row with None key") + logger.info(dataclass_type) for key, value in row.items(): - print(f"key: {key} value: {value}") + logger.info(f"key: {key} value: {value}") + TerminalHelper.prompt_for_execution(False, "COnintue?", "DEBUG") continue row_id = row[id_field] @@ -914,3 +916,16 @@ class ExtraTransitionDomain: # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} return dict_data + # def clean_delimiters(self, data_row) -> (str, bool): + # """ This function was created to prevent errors where data files had spaces + # erroneously injected around the delimiters. """ + + # cleaning_was_needed = False + + + # TerminalHelper.print_conditional(cleaning_was_needed, + # (f"{TerminalColors.YELLOW}" + # f"WARNING: Data file has spaces" + # f"around the delimiters. Removing" + # f"erroneous spaces..." + # f"{TerminalColors.ENDC}")) \ No newline at end of file From ba9ba10131836bdd6138f9db2b11a9cb93630626 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:31:54 -0700 Subject: [PATCH 50/88] Update script --- .../commands/load_transition_domain.py | 52 ++++++++++++++----- .../commands/master_domain_migrations.py | 2 +- .../utility/extra_transition_domain_helper.py | 19 +++++-- .../test_transition_domain_migrations.py | 3 ++ 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 7edf22457..7bf5edc66 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -324,6 +324,7 @@ class Command(BaseCommand): **options, ): """Parse the data files and create TransitionDomains.""" + args = TransitionDomainArguments(**options) ### Process JSON file ### # If a JSON was provided, use its values instead of defaults. # TODO: there is no way to discern user overrides from those arg’s defaults. @@ -332,7 +333,7 @@ class Command(BaseCommand): try: data = json.load(jsonFile) # Create an instance of TransitionDomainArguments - args = TransitionDomainArguments() + # Iterate over the data from the JSON file for key, value in data.items(): # Check if the key exists in TransitionDomainArguments @@ -346,32 +347,57 @@ class Command(BaseCommand): """) raise err - sep = options.get("sep") + sep = args.sep # If --resetTable was used, prompt user to confirm # deletion of table data - if options.get("resetTable"): + if args.resetTable: self.prompt_table_reset() # Get --debug argument - debug_on = options.get("debug") + debug_on = args.debug # Get --LimitParse argument debug_max_entries_to_parse = int( - options.get("limitParse") + args.limitParse ) # set to 0 to parse all entries ## Variables for Additional TransitionDomain Information ## # Desired directory for additional TransitionDomain data # (In the event they are stored seperately) - directory = options.get("directory") + directory = args.directory + # Add a slash if the last character isn't one + if directory and directory[-1] != "/": + directory += "/" - # Main script filenames - # TODO: @ZANDER to replace this with new TransitionDomainArgument object - domain_contacts_filename = directory + "/" + options.get("domain_contacts_filename") - contacts_filename = directory + "/" + options.get("contacts_filename") - domain_statuses_filename = directory + "/" + options.get("domain_statuses_filename") + # Main script filenames - these do not have defaults + domain_contacts_filename = None + try: + domain_contacts_filename = directory + options.get("domain_contacts_filename") + except TypeError as err: + logger.error( + f"Invalid filename of '{args.domain_contacts_filename}'" + " was provided for domain_contacts_filename" + ) + + contacts_filename = None + try: + contacts_filename = directory + options.get("contacts_filename") + except TypeError as err: + logger.error( + f"Invalid filename of '{args.contacts_filename}'" + " was provided for contacts_filename" + ) + + domain_statuses_filename = None + try: + domain_statuses_filename = directory + options.get("domain_statuses_filename") + except TypeError as err: + logger.error( + f"Invalid filename of '{args.domain_statuses_filename}'" + " was provided for domain_statuses_filename" + ) # Agency information agency_adhoc_filename = options.get("agency_adhoc_filename") @@ -593,7 +619,7 @@ class Command(BaseCommand): # Prompt the user if they want to load additional data on the domains title = "Do you wish to load additional data for TransitionDomains?" - do_parse_extra = TerminalHelper.prompt_for_execution( + proceed = TerminalHelper.prompt_for_execution( system_exit_on_terminate=False, info_to_inspect=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING @@ -617,6 +643,6 @@ class Command(BaseCommand): """, prompt_title=title, ) - if do_parse_extra: + if proceed: arguments = TransitionDomainArguments(**options) self.parse_extra(arguments) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 58124157f..557a2e125 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -294,7 +294,7 @@ class Command(BaseCommand): command_script = "load_transition_domain" command_string = ( f"./manage.py {command_script} " - f"{file_location+migration_json_filename}" + f"{file_location+migration_json_filename} " ) if sep is not None and sep != "|": command_string += f"--sep {sep} " diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7022944a0..291ce9c1f 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -176,6 +176,7 @@ class LoadExtraTransitionDomain: # Check if the instance has changed before saving #if updated_transition_domain.__dict__ != transition_domain.__dict__: + updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) if self.debug: @@ -214,6 +215,7 @@ class LoadExtraTransitionDomain: """ ) else: + # TODO - update logger.error( f"""{TerminalColors.FAIL} ============= FINISHED WITH ERRORS =============== @@ -222,6 +224,9 @@ class LoadExtraTransitionDomain: {TerminalColors.ENDC} """ ) + # TODO + if TransitionDomain.objects.all().count() != len(updated_transition_domains): + logger.error("Something bad happened") def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it @@ -517,7 +522,6 @@ class LoadExtraTransitionDomain: def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, based off a desired_id""" - l = self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id.lower()) return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: @@ -545,6 +549,7 @@ class LoadExtraTransitionDomain: based off a desired_id""" return self.get_object_by_id(EnumFilenames.DOMAIN_ESCROW, desired_id) + # TODO - renamed / needs a return section def get_object_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. @@ -591,7 +596,7 @@ class LoadExtraTransitionDomain: ) return obj - +# TODO - change name @dataclass class PatternMap: """Helper class that holds data and metadata about a requested file. @@ -612,7 +617,6 @@ class PatternMap: id_field: data_type(...), ... } - """ def __init__( @@ -635,6 +639,7 @@ class PatternMap: self.data_type = data_type ### What the id should be in the holding dict ### + # TODO - rename to id_field_name self.id_field = id_field # Object data # @@ -743,9 +748,17 @@ class ExtraTransitionDomain: AuthorityAdhoc, "authorityid", ), + ( + EnumFilenames.AUTHORITY_ADHOC, + options.authority_adhoc_filename, + AuthorityAdhoc, + "authorityid", + ), ] + self.file_data = self.populate_file_data(pattern_map_params) + # TODO - revise comment def populate_file_data( self, pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] ): diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index ef74f8744..665a0aad9 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -352,6 +352,8 @@ class TestMigrations(TestCase): ) ] + + #TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") # Afterwards, their values should be what we expect all_transition_domains = TransitionDomain.objects.all() for domain in all_transition_domains: @@ -433,6 +435,7 @@ class TestMigrations(TestCase): expected.id = domain.id expected.created_at = domain.created_at expected.updated_at = domain.updated_at + self.assertEqual(domain, expected) def test_transfer_transition_domains_to_domains(self): From a0ce0c137aa6605fc563c6517cb7043d58ddd6e6 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 18:47:35 -0600 Subject: [PATCH 51/88] fix error where disabling prompts means not loading data --- .../management/commands/master_domain_migrations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 557a2e125..27fdcf624 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -315,7 +315,7 @@ class Command(BaseCommand): ) # TODO: make this somehow run inside TerminalHelper prompt - if proceed: + if proceed or not prompts_enabled: call_command( command_script, f"{file_location+migration_json_filename}", @@ -342,7 +342,7 @@ class Command(BaseCommand): "Running transfer_transition_domains_to_domains script", ) # TODO: make this somehow run inside TerminalHelper prompt - if proceed: + if proceed or not prompts_enabled: call_command(command_script) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): @@ -360,7 +360,7 @@ class Command(BaseCommand): ) # TODO: make this somehow run inside TerminalHelper prompt - if proceed: + if proceed or not prompts_enabled: call_command(command_script, send_emails=True) def run_migration_scripts( From 4a7aa0b484780ec46b790e90517c4031dc13ec44 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:50:46 -0700 Subject: [PATCH 52/88] Add comment --- src/registrar/views/utility/mixins.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 9d99c48b0..1e43a3e24 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -100,6 +100,9 @@ class DomainPermission(PermissionsLoginMixin): if DomainInformation.objects.filter(id=pk).exists(): requested_domain = DomainInformation.objects.get(id=pk) + # If no domain_application object exists and we are + # coming from the manage_domain dashboard, this is likely + # a transition domain. domain_application = requested_domain.domain_application if not hasattr(domain_application, "status"): return True From 87254543f43cbcf7e16c7b54a146468296424ff0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 7 Nov 2023 18:21:48 -0700 Subject: [PATCH 53/88] Fix logging --- .../utility/extra_transition_domain_helper.py | 26 ++++++++--- .../test_transition_domain_migrations.py | 44 +++++++++---------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 0952e8f41..ee44d31cf 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -71,7 +71,7 @@ class FileTransitionLog: def create_log_item( - self, file_type, code, message, domain_name=None, add_to_list=True + self, file_type, code, message, domain_name=None, add_to_list=True, minimal_logging=True ): """Creates and returns an LogItem object. @@ -83,6 +83,15 @@ class FileTransitionLog: dict_name = (file_type, domain_name) self._add_to_log_list(dict_name, log) + + restrict_type = [] + if minimal_logging: + restrict_type = [LogCode.INFO, LogCode.WARNING] + TerminalHelper.print_conditional( + log.code not in restrict_type, + log.message, + log.code, + ) return log @@ -179,9 +188,7 @@ class LoadExtraTransitionDomain: updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) - if self.debug: - # Display errors for this specific domain - self.parse_logs.display_logs_by_domain_name(domain_name) + logger.info( f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" @@ -242,6 +249,7 @@ class LoadExtraTransitionDomain: "Could not add epp_creation_date and epp_expiration_date " f"on {domain_name}, no data exists.", domain_name, + not self.debug ) return transition_domain @@ -287,6 +295,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add federal_agency on {domain_name}, no data exists.", domain_name, + not self.debug ) return transition_domain @@ -301,6 +310,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add inactive agency {info.agencyname} on {domain_name}", domain_name, + not self.debug ) return transition_domain @@ -310,6 +320,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add non-federal agency {info.agencyname} on {domain_name}", domain_name, + not self.debug ) return transition_domain @@ -342,6 +353,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add domain_type on {domain_name}, no data exists.", domain_name, + not self.debug ) return transition_domain @@ -364,6 +376,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", domain_name, + not self.debug ) return transition_domain @@ -424,6 +437,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add organization_name on {domain_name}, no data exists.", domain_name, + not self.debug ) return transition_domain @@ -457,6 +471,7 @@ class LoadExtraTransitionDomain: LogCode.INFO, f"Added {var_name} as '{changed_value}' on {domain_name}", domain_name, + not self.debug ) else: self.parse_logs.create_log_item( @@ -464,6 +479,7 @@ class LoadExtraTransitionDomain: LogCode.WARNING, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", domain_name, + not self.debug ) # Property getters, i.e. orgid or domaintypeid @@ -583,7 +599,7 @@ class LoadExtraTransitionDomain: desired_type = self.parsed_data_container.file_data.get(file_type) if desired_type is None: self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Type {file_type} does not exist" + file_type, LogCode.ERROR, f"Type {file_type} does not exist", ) return None diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 665a0aad9..6a7e36942 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -394,6 +394,25 @@ class TestMigrations(TestCase): expected_missing_domain_invitations, ) + expected_anomaly_domains = Domain.objects.filter(name="anomaly.gov") + self.assertEqual(expected_anomaly_domains.count(), 1) + expected_anomaly = expected_anomaly_domains.get() + + self.assertEqual(expected_anomaly.expiration_date, "test") + self.assertEqual(expected_anomaly.creation_date, "test") + self.assertEqual(expected_anomaly.name, "anomaly.gov") + self.assertEqual(expected_anomaly.state, "ready") + + expected_testdomain_domains = Domain.objects.filter(name="anomaly.gov") + self.assertEqual(expected_testdomain_domains.count(), 1) + + expected_testdomain = expected_testdomain_domains.get() + + self.assertEqual(expected_testdomain.expiration_date, "test") + self.assertEqual(expected_testdomain.creation_date, "test") + self.assertEqual(expected_testdomain.name, "anomaly.gov") + self.assertEqual(expected_testdomain.state, "ready") + expected_domains = [ Domain( expiration_date=None, @@ -405,32 +424,9 @@ class TestMigrations(TestCase): name="testdomain.gov", state="ready", ), - Domain( - expiration_date=None, - name="fakewebsite1.gov", - state="on hold", - ), - Domain( - expiration_date=None, - name="fakewebsite2.gov", - state="on hold", - ), - Domain( - expiration_date=None, - name="fakewebsite3.gov", - state="ready", - ), ] - + for domain in Domain.objects.all(): - print(f""" - Domain( - expiration_date={domain.expiration_date}, - name="{domain.name}", - state="{domain.state}", - ), - """ - ) for expected in expected_domains: expected.id = domain.id expected.created_at = domain.created_at From 9e0d8a49a77fa3a1d3ceaf008b4e6a8247906860 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 19:21:48 -0600 Subject: [PATCH 54/88] Fix for delimiter whitespace issue --- .../utility/extra_transition_domain_helper.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 0952e8f41..0ba6a8f1d 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -894,6 +894,8 @@ class ExtraTransitionDomain: dict_data = {} with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.reader(requested_file, delimiter=seperator) + # clean the rows of any whitespace around delimiters + for row in reader: yield (c.strip() for c in row) for row in reader: domain_name = row[0] date_format = "%Y-%m-%dT%H:%M:%SZ" @@ -908,10 +910,11 @@ class ExtraTransitionDomain: def _read_csv_file(self, file, seperator, dataclass_type, id_field): with open(file, "r", encoding="utf-8-sig") as requested_file: - reader = csv.DictReader(requested_file, skipinitialspace=True, delimiter=seperator) + reader = csv.DictReader(requested_file, delimiter=seperator) dict_data = {} + # clean the rows of any whitespace around delimiters + for row in reader: yield (c.strip() for c in row) for row in reader: - # cleaned_row, cleaning_was_needed = self.clean_delimiters() if None in row: logger.info("Skipping row with None key") logger.info(dataclass_type) @@ -927,18 +930,4 @@ class ExtraTransitionDomain: row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} - return dict_data - - # def clean_delimiters(self, data_row) -> (str, bool): - # """ This function was created to prevent errors where data files had spaces - # erroneously injected around the delimiters. """ - - # cleaning_was_needed = False - - - # TerminalHelper.print_conditional(cleaning_was_needed, - # (f"{TerminalColors.YELLOW}" - # f"WARNING: Data file has spaces" - # f"around the delimiters. Removing" - # f"erroneous spaces..." - # f"{TerminalColors.ENDC}")) \ No newline at end of file + return dict_data \ No newline at end of file From 22f9952ffd2f76ae2bc76e270d3a16b30edb4e0f Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Nov 2023 19:35:22 -0600 Subject: [PATCH 55/88] update debug statements --- src/registrar/management/commands/load_transition_domain.py | 2 +- .../commands/utility/extra_transition_domain_helper.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 7bf5edc66..1744807bb 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -468,7 +468,7 @@ class Command(BaseCommand): new_entry_email = "" new_entry_emailSent = False # set to False by default - TerminalHelper.print_conditional(debug_on, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") + TerminalHelper.print_conditional(True, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") # PART 1: Get the status if new_entry_domain_name not in domain_status_dictionary: diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 97068a2ef..5cea91204 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -213,16 +213,17 @@ class LoadExtraTransitionDomain: failed_count = len(failed_transition_domains) if failed_count == 0: + TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Updated {len(updated_transition_domains)} transition domain entries: - {TerminalHelper.array_as_string(updated_transition_domains)} {TerminalColors.ENDC} """ ) else: # TODO - update + TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") logger.error( f"""{TerminalColors.FAIL} ============= FINISHED WITH ERRORS =============== From 78658653a6b945a040fdf8e0e07bf3e93e1436d3 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:45:07 -0700 Subject: [PATCH 56/88] Fix parse error due to corrupt data --- .../utility/extra_transition_domain_helper.py | 71 ++++++++++++--- .../test_transition_domain_migrations.py | 87 +------------------ 2 files changed, 60 insertions(+), 98 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 5cea91204..552997f10 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -2,7 +2,7 @@ import csv from dataclasses import dataclass from datetime import datetime -from enum import Enum +import io import glob import re import logging @@ -911,8 +911,6 @@ class ExtraTransitionDomain: dict_data = {} with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.reader(requested_file, delimiter=seperator) - # clean the rows of any whitespace around delimiters - for row in reader: yield (c.strip() for c in row) for row in reader: domain_name = row[0] date_format = "%Y-%m-%dT%H:%M:%SZ" @@ -926,19 +924,24 @@ class ExtraTransitionDomain: return dict_data def _read_csv_file(self, file, seperator, dataclass_type, id_field): + dict_data = {} + # Used when we encounter bad data + updated_file_content = None with open(file, "r", encoding="utf-8-sig") as requested_file: reader = csv.DictReader(requested_file, delimiter=seperator) - dict_data = {} - # clean the rows of any whitespace around delimiters - for row in reader: yield (c.strip() for c in row) for row in reader: + # Checks if we encounter any bad data. + # If we do, we (non-destructively) clean the file if None in row: - logger.info("Skipping row with None key") - logger.info(dataclass_type) - for key, value in row.items(): - logger.info(f"key: {key} value: {value}") - TerminalHelper.prompt_for_execution(False, "COnintue?", "DEBUG") - continue + logger.warning( + f"{TerminalColors.YELLOW}" + f"Found bad data in {file}. Attempting to clean." + f"{TerminalColors.ENDC}" + ) + updated_file_content = self.replace_bad_seperators(file, f"{seperator}", ";badseperator;") + dict_data = {} + break + row_id = row[id_field] # To maintain pairity with the load_transition_domain @@ -946,5 +949,45 @@ class ExtraTransitionDomain: if id_field == "domainname" and row_id is not None: row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) - # dict_data = {row[id_field]: dataclass_type(**row) for row in reader} - return dict_data \ No newline at end of file + + # After we clean the data, try to parse it again + if updated_file_content: + logger.info( + f"{TerminalColors.MAGENTA}" + f"Retrying load for {file}" + f"{TerminalColors.ENDC}" + ) + # Store the file locally rather than writing to the file. + # This is to avoid potential data corruption. + updated_file = io.StringIO(updated_file_content) + reader = csv.DictReader(updated_file, delimiter=seperator) + for row in reader: + row_id = row[id_field] + # If the key is still none, something + # is wrong with the file. + if None in row: + logger.error( + f"{TerminalColors.FAIL}" + f"Corrupt data found for {row_id}. Skipping." + f"{TerminalColors.ENDC}" + ) + continue + + for key, value in row.items(): + if value is not None and isinstance(value, str): + value = value.replace(";badseperator;", f" {seperator} ") + row[key] = value + + # To maintain pairity with the load_transition_domain + # script, we store this data in lowercase. + if id_field == "domainname" and row_id is not None: + row_id = row_id.lower() + dict_data[row_id] = dataclass_type(**row) + return dict_data + + def replace_bad_seperators(self, filename, delimiter, special_character): + with open(filename, "r", encoding="utf-8-sig") as file: + contents = file.read() + + new_content = re.sub(rf" \{delimiter} ", special_character, contents) + return new_content \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 6a7e36942..f14f42545 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -242,66 +242,6 @@ class TestMigrations(TestCase): ) expected_transition_domains = [ - TransitionDomain( - username="", - domain_name="anomaly.gov", - status="ready", - email_sent=False, - organization_type=None, - organization_name="Flashdog", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None - ), - TransitionDomain( - username="testuser@gmail.com", - domain_name="testdomain.gov", - status="ready", - email_sent=False, - organization_type=None, - organization_name="Gigaclub", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None - ), - TransitionDomain( - username="agustina.wyman7@test.com", - domain_name="fakewebsite1.gov", - status="on hold", - email_sent=False, - organization_type=None, - organization_name="Midel", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None - ), - TransitionDomain( - username="susy.martin4@test.com", - domain_name="fakewebsite1.gov", - status="on hold", - email_sent=False, - organization_type=None, - organization_name="Midel", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None - ), - TransitionDomain( - username="stephania.winters4@test.com", - domain_name="fakewebsite1.gov", - status="on hold", - email_sent=False, - organization_type=None, - organization_name="Midel", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None - ), TransitionDomain( username="alexandra.bobbitt5@test.com", domain_name="fakewebsite2.gov", @@ -314,30 +254,6 @@ class TestMigrations(TestCase): epp_creation_date=None, epp_expiration_date=None ), - TransitionDomain( - username="jospeh.mcdowell3@test.com", - domain_name="fakewebsite2.gov", - status="on hold", - email_sent=False, - organization_type="Federal", - organization_name="Fanoodle", - federal_type="Executive", - federal_agency="InnoZ", - epp_creation_date=None, - epp_expiration_date=None - ), - TransitionDomain( - username="reginald.ratcliff4@test.com", - domain_name="fakewebsite2.gov", - status="on hold", - email_sent=False, - organization_type="Federal", - organization_name="Fanoodle", - federal_type="Executive", - federal_agency="InnoZ", - epp_creation_date=None, - epp_expiration_date=None - ), TransitionDomain( username="reginald.ratcliff4@test.com", domain_name="fakewebsite3.gov", @@ -352,6 +268,9 @@ class TestMigrations(TestCase): ) ] + expected_transition_domains = TransitionDomain.objects.filter(username="alexandra.bobbitt5@test.com") + self.assertEqual(expected_transition_domains.count(), 1) + expected_transition_domain = expected_transition_domains.get() #TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") # Afterwards, their values should be what we expect From a2f0be72ce8eb276af869e7d13604b5571750884 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:54:57 -0700 Subject: [PATCH 57/88] Readd changed pip --- src/Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Pipfile b/src/Pipfile index 33f2c0954..d4a9bbafb 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [packages] django = "*" cfenv = "*" +django-cors-headers = "*" pycryptodomex = "*" django-allow-cidr = "*" django-auditlog = "*" From 2c49ea97ef98f7378c1c9d18a2a02f46ce46ae9f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:40:09 -0700 Subject: [PATCH 58/88] Fix merge issues and test --- .../runbooks/rotate_application_secrets.md | 31 ++++++++--- src/registrar/assets/sass/_theme/_admin.scss | 2 +- src/registrar/assets/sass/_theme/_alerts.scss | 6 +- src/registrar/config/settings.py | 12 +++- src/registrar/templates/base.html | 7 ++- .../django/admin/domain_change_list.html | 2 +- .../tests/data/test_migrationFilepaths.json | 12 ++++ .../tests/data/test_organization_adhoc.txt | 2 +- .../test_transition_domain_migrations.py | 55 ++++++++----------- src/registrar/utility/csv_export.py | 26 ++++++++- 10 files changed, 102 insertions(+), 53 deletions(-) create mode 100644 src/registrar/tests/data/test_migrationFilepaths.json diff --git a/docs/operations/runbooks/rotate_application_secrets.md b/docs/operations/runbooks/rotate_application_secrets.md index e91e8427e..78c402efe 100644 --- a/docs/operations/runbooks/rotate_application_secrets.md +++ b/docs/operations/runbooks/rotate_application_secrets.md @@ -76,16 +76,24 @@ These are the client certificate and its private key used to identify the regist The private key is protected by a passphrase for safer transport and storage. -These were generated with: +These were generated with the following steps: + +### Step 1: Generate an unencrypted private key with a named curve ```bash -openssl genpkey -out client.key \ - -algorithm EC -pkeyopt ec_paramgen_curve:P-256 \ - -aes-256-cbc -openssl req -new -x509 -days 365 \ - -key client.key -out client.crt \ - -subj "/C=US/ST=DC/L=Washington/O=GSA/OU=18F/CN=GOV Prototype Registrar" +openssl ecparam -name prime256v1 -genkey -out client_unencrypted.key +``` +### Step 2: Create an encrypted private key with a passphrase + +```bash +openssl pkcs8 -topk8 -v2 aes-256-cbc -in client_unencrypted.key -out client.key +``` + +### Generate the certificate + +```bash +openssl req -new -x509 -days 365 -key client.key -out client.crt -subj "/C=US/ST=DC/L=Washington/O=GSA/OU=18F/CN=GOV Prototype Registrar" ``` (If you can't use openssl on your computer directly, you can access it using Docker as `docker run --platform=linux/amd64 -it --rm -v $(pwd):/apps -w /apps alpine/openssl`.) @@ -97,7 +105,14 @@ base64 client.key base64 client.crt ``` -You'll need to give the new certificate to the registry vendor _before_ rotating it in production. +Note depending on your system you may need to instead run: + +```bash +base64 -i client.key +base64 -i client.crt +``` + +You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vender, make sure to update the kdbx file on Google Drive. ## REGISTRY_HOSTNAME diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 68ff51597..080a343ee 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -187,7 +187,7 @@ h1, h2, h3 { .object-tools li a, .object-tools p a { font-family: "Source Sans Pro Web", "Helvetica Neue", Helvetica, Roboto, Arial, sans-serif; - text-transform: capitalize!important; + text-transform: none!important; font-size: 14px!important; } diff --git a/src/registrar/assets/sass/_theme/_alerts.scss b/src/registrar/assets/sass/_theme/_alerts.scss index dd51734ed..9ee28a357 100644 --- a/src/registrar/assets/sass/_theme/_alerts.scss +++ b/src/registrar/assets/sass/_theme/_alerts.scss @@ -5,7 +5,11 @@ font-size: units(3); } } - + +.usa-alert__text.measure-none { + max-width: measure(none); +} + // The icon was off center for some reason // Fixes that issue @media (min-width: 64em){ diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index e10b8584c..896691efb 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -46,7 +46,6 @@ path = Path(__file__) env_db_url = env.dj_db_url("DATABASE_URL") env_debug = env.bool("DJANGO_DEBUG", default=False) -env_is_production = env.bool("IS_PRODUCTION", default=False) env_log_level = env.str("DJANGO_LOG_LEVEL", "DEBUG") env_base_url = env.str("DJANGO_BASE_URL") env_getgov_public_site_url = env.str("GETGOV_PUBLIC_SITE_URL", "") @@ -74,8 +73,6 @@ BASE_DIR = path.resolve().parent.parent.parent # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env_debug -# Controls production specific feature toggles -IS_PRODUCTION = env_is_production # Applications are modular pieces of code. # They are provided by Django, by third-parties, or by yourself. @@ -122,6 +119,8 @@ INSTALLED_APPS = [ "api", # Only for generating documentation, uncomment to run manage.py generate_puml # "puml_generator", + # supports necessary headers for Django cross origin + "corsheaders", ] # Middleware are routines for processing web requests. @@ -130,6 +129,8 @@ INSTALLED_APPS = [ MIDDLEWARE = [ # django-allow-cidr: enable use of CIDR IP ranges in ALLOWED_HOSTS "allow_cidr.middleware.AllowCIDRMiddleware", + # django-cors-headers: listen to cors responses + "corsheaders.middleware.CorsMiddleware", # serve static assets in production "whitenoise.middleware.WhiteNoiseMiddleware", # provide security enhancements to the request/response cycle @@ -295,6 +296,11 @@ CSP_DEFAULT_SRC = allowed_sources CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources +# Cross-Origin Resource Sharing (CORS) configuration +# Sets clients that allow access control to manage.get.gov +# TODO: remove :8080 to see if we can have all localhost access +CORS_ALLOWED_ORIGINS = ["http://localhost:8080", "https://beta.get.gov"] + # Content-Length header is set by django.middleware.common.CommonMiddleware diff --git a/src/registrar/templates/base.html b/src/registrar/templates/base.html index 72c30f323..68a5c69ea 100644 --- a/src/registrar/templates/base.html +++ b/src/registrar/templates/base.html @@ -62,10 +62,11 @@ {% if IS_DEMO_SITE %}
    -
    +
    -

    - BETA SITE: We’re building a new way to get a .gov. Take a look around, but don’t rely on this site yet. This site is for testing purposes only. Don’t enter real data into any form on this site. To learn about requesting a .gov domain, visit get.gov +

    New domain requests are paused

    +

    + This is the new registrar for managing .gov domains. Note that we’re not accepting requests for new .gov domains until January 2024. Follow .gov updates at get.gov/updates/.

    diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index 68fdbe7aa..22df74685 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -15,7 +15,7 @@ {% if has_add_permission %}
  • - Add Domain + Add domain
    • {% endif %} diff --git a/src/registrar/tests/data/test_migrationFilepaths.json b/src/registrar/tests/data/test_migrationFilepaths.json new file mode 100644 index 000000000..605f02f92 --- /dev/null +++ b/src/registrar/tests/data/test_migrationFilepaths.json @@ -0,0 +1,12 @@ +{ + "directory": "registrar/tests/data", + "agency_adhoc_filename": "test_agency_adhoc.txt", + "authority_adhoc_filename": "test_authority_adhoc.txt", + "organization_adhoc_filename": "test_organization_adhoc.txt", + "domain_adhoc_filename": "test_domain_types_adhoc.txt", + "domain_additional_filename": "test_domain_additional.txt", + "domain_contacts_filename": "test_domain_contacts.txt", + "domain_escrow_filename": "test_escrow_domains_daily.txt", + "domain_statuses_filename": "test_domain_statuses.txt", + "contacts_filename": "test_contacts.txt" +} \ No newline at end of file diff --git a/src/registrar/tests/data/test_organization_adhoc.txt b/src/registrar/tests/data/test_organization_adhoc.txt index a39361a10..43ad637b2 100644 --- a/src/registrar/tests/data/test_organization_adhoc.txt +++ b/src/registrar/tests/data/test_organization_adhoc.txt @@ -1,6 +1,6 @@ orgid|orgname|orgstreet|orgcity|orgstate|orgzip|orgcountrycode 1|Flashdog|298 Monument Hill|Lakeland|Florida|33805|US 2|Gigaclub|782 Mosinee Lane|Alexandria|Louisiana|71307|US -3|Midel|376 Joe Pass|Waco|Texas|76705|US +3|corrupt data|376 Joe Pass|Waco | corruption|Texas|76705|US 4|Fanoodle|93001 Arizona Drive|Columbus|Ohio|43268|US 5|Sushi|9999 Sushi Way|Columbus|Ohio|43268|US \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index f14f42545..7781e66e9 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -1,4 +1,4 @@ -from io import StringIO +import datetime from django.test import TestCase from registrar.models import ( @@ -20,7 +20,7 @@ class TestMigrations(TestCase): # self.transfer_script = "transfer_transition_domains_to_domains", # self.master_script = "load_transition_domain", - self.test_data_file_location = "/app/registrar/tests/data" + self.test_data_file_location = "registrar/tests/data" self.test_domain_contact_filename = "test_domain_contacts.txt" self.test_contact_filename = "test_contacts.txt" self.test_domain_status_filename = "test_domain_statuses.txt" @@ -32,6 +32,7 @@ class TestMigrations(TestCase): self.test_domain_types_adhoc = "test_domain_types_adhoc.txt" self.test_escrow_domains_daily = "test_escrow_domains_daily" self.test_organization_adhoc = "test_organization_adhoc.txt" + self.migration_json_filename = "test_migrationFilepaths.json" def tearDown(self): # Delete domain information @@ -48,16 +49,8 @@ class TestMigrations(TestCase): with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): call_command( "load_transition_domain", - f"{self.test_data_file_location}/{self.test_domain_contact_filename}", - f"{self.test_data_file_location}/{self.test_contact_filename}", - f"{self.test_data_file_location}/{self.test_domain_status_filename}", - directory=self.test_data_file_location, - agency_adhoc_filename=self.test_agency_adhoc_filename, - domain_additional_filename=self.test_domain_additional, - domain_escrow_filename=self.test_escrow_domains_daily, - domain_adhoc_filename=self.test_domain_types_adhoc, - organization_adhoc_filename=self.test_organization_adhoc, - authority_adhoc_filename=self.test_authority_adhoc_filename, + f"{self.test_data_file_location}/{self.migration_json_filename}", + directory=self.test_data_file_location ) def run_transfer_domains(self): @@ -68,12 +61,8 @@ class TestMigrations(TestCase): call_command( "master_domain_migrations", runMigrations=True, - migrationDirectory=f"{self.test_data_file_location}", - migrationFilenames=( - f"{self.test_domain_contact_filename}," - f"{self.test_contact_filename}," - f"{self.test_domain_status_filename}" - ), + migrationDirectory=self.test_data_file_location, + migration_json_filename=self.migration_json_filename, ) def compare_tables( @@ -313,24 +302,26 @@ class TestMigrations(TestCase): expected_missing_domain_invitations, ) - expected_anomaly_domains = Domain.objects.filter(name="anomaly.gov") - self.assertEqual(expected_anomaly_domains.count(), 1) - expected_anomaly = expected_anomaly_domains.get() + anomaly_domains = Domain.objects.filter(name="anomaly.gov") + self.assertEqual(anomaly_domains.count(), 1) + anomaly = anomaly_domains.get() - self.assertEqual(expected_anomaly.expiration_date, "test") - self.assertEqual(expected_anomaly.creation_date, "test") - self.assertEqual(expected_anomaly.name, "anomaly.gov") - self.assertEqual(expected_anomaly.state, "ready") + self.assertEqual(anomaly.expiration_date, datetime.date(2023, 3, 9)) + self.assertEqual( + anomaly.created_at, datetime.datetime(2023, 11, 8, 17, 23, 46, 764663, tzinfo=datetime.timezone.utc) + ) + self.assertEqual(anomaly.name, "anomaly.gov") + self.assertEqual(anomaly.state, "ready") - expected_testdomain_domains = Domain.objects.filter(name="anomaly.gov") - self.assertEqual(expected_testdomain_domains.count(), 1) + testdomain_domains = Domain.objects.filter(name="testdomain.gov") + self.assertEqual(testdomain_domains.count(), 1) - expected_testdomain = expected_testdomain_domains.get() + testdomain = testdomain_domains.get() - self.assertEqual(expected_testdomain.expiration_date, "test") - self.assertEqual(expected_testdomain.creation_date, "test") - self.assertEqual(expected_testdomain.name, "anomaly.gov") - self.assertEqual(expected_testdomain.state, "ready") + self.assertEqual(testdomain.expiration_date, datetime.date(2023, 3, 9)) + self.assertEqual(testdomain.created_at, "test") + self.assertEqual(testdomain.name, "anomaly.gov") + self.assertEqual(testdomain.state, "ready") expected_domains = [ Domain( diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index ffada0a0b..d9127d72c 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,6 +2,8 @@ import csv from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact +from django.db.models import Value +from django.db.models.functions import Coalesce def export_domains_to_writer(writer, columns, sort_fields, filter_condition): @@ -61,7 +63,13 @@ def export_data_type_to_csv(csv_file): "Status", "Expiration Date", ] - sort_fields = ["domain__name"] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] filter_condition = { "domain__state__in": [ Domain.State.READY, @@ -84,7 +92,13 @@ def export_data_full_to_csv(csv_file): "State", "Security Contact Email", ] - sort_fields = ["domain__name", "federal_agency", "organization_type"] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] filter_condition = { "domain__state__in": [ Domain.State.READY, @@ -107,7 +121,13 @@ def export_data_federal_to_csv(csv_file): "State", "Security Contact Email", ] - sort_fields = ["domain__name", "federal_agency", "organization_type"] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] filter_condition = { "organization_type__icontains": "federal", "domain__state__in": [ From 025f08f3d0fa7ae0c0f52c11f5d4c71fbb0ca4f0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:50:05 -0700 Subject: [PATCH 59/88] Fix addl Merge issues --- docs/operations/data_migration.md | 28 +++++++++++++++++----------- src/migrationdata/README.md | 4 ++-- src/registrar/config/settings.py | 3 +++ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 2dd00a39f..75bd50a9b 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -211,11 +211,12 @@ This will allow Docker to mount the files to a container (under `/app`) for our ## Transition Domains (Part 2) - Running the Migration Scripts -*NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.* ### STEP 1: Load Transition Domains -Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory) +Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. + +(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` @@ -237,6 +238,8 @@ This will delete all the data in transtion_domain. It is helpful if you want to Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): + +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` @@ -251,9 +254,11 @@ Directs the script to load only the first 100 entries into the table. You can a ### STEP 3: Send Domain invitations -To send invitations for every transition domain in the transition domain table, execute the following command: +To send invitation emails for every transition domain in the transition domain table, execute the following command: + +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell -docker compose run -T app send_domain_invitations -s +docker compose run -T app ./manage.py send_domain_invitations -s ``` ### STEP 4: Test the results (Run the analyzer script) @@ -263,6 +268,8 @@ This script's main function is to scan the transition domain and domain tables f #### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: + +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` @@ -270,6 +277,8 @@ docker compose run -T app ./manage.py master_domain_migrations --debug #### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: + +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt ``` @@ -278,13 +287,12 @@ docker compose run -T app ./manage.py master_domain_migrations --runMigrations - `--runMigrations` -A boolean (default to true), which triggers running -all scripts (in sequence) for transition domain migrations +Runs all scripts (in sequence) for transition domain migrations `--migrationDirectory` The location of the files used for load_transition_domain migration script. -(default is "migrationData" (This is the sandbox directory)) +(default is "migrationdata" (This is the sandbox directory)) Example Usage: *--migrationDirectory /app/tmp* @@ -310,13 +318,11 @@ Delimiter for the migration scripts to correctly parse the given text files. `--debug` -A boolean (default to true), which activates additional print statements +Activates additional print statements `--prompt` -A boolean (default to true), which activates terminal prompts -that allows the user to step through each portion of this -script. +Activates terminal prompts that allows the user to step through each portion of this script. `--limitParse` diff --git a/src/migrationdata/README.md b/src/migrationdata/README.md index 585624bdb..81190ee3f 100644 --- a/src/migrationdata/README.md +++ b/src/migrationdata/README.md @@ -1,8 +1,8 @@ ## Purpose -Use this folder for storing files for the migration process. Should otherwise be empty on local dev environments unless necessary. This folder must exist due to the nature of how data is stored on cloud.gov and the nature of the data we typically want to send. +Use this folder for storing files for the migration process. Should otherwise be empty on local dev environments unless necessary. This folder must exist due to the nature of how data is stored on cloud.gov and the nature of the data we want to send. ## How do I migrate registrar data? This process is detailed in [data_migration.md](../../docs/operations/data_migration.md) ## What kind of files can I store here? -The intent is for PII data or otherwise, but this can exist in any format. Do note that the data contained in this file will be temporary, so after the app is restaged it will lose it (as long as nothing is committed). This is ideal for migration files as they write to our DB, but not for something you need to permanently hold onto. \ No newline at end of file +The intent is for PII data or otherwise, but this can exist in any format. Do note that the data contained in this file will be temporary, so after the app is restaged it will lose it. This is ideal for migration files as they write to our DB, but not for something you need to permanently hold onto. \ No newline at end of file diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 896691efb..87ad0ff96 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -46,6 +46,7 @@ path = Path(__file__) env_db_url = env.dj_db_url("DATABASE_URL") env_debug = env.bool("DJANGO_DEBUG", default=False) +env_is_production = env.bool("IS_PRODUCTION", default=False) env_log_level = env.str("DJANGO_LOG_LEVEL", "DEBUG") env_base_url = env.str("DJANGO_BASE_URL") env_getgov_public_site_url = env.str("GETGOV_PUBLIC_SITE_URL", "") @@ -73,6 +74,8 @@ BASE_DIR = path.resolve().parent.parent.parent # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env_debug +# Controls production specific feature toggles +IS_PRODUCTION = env_is_production # Applications are modular pieces of code. # They are provided by Django, by third-parties, or by yourself. From 0bc03047b81ecc67ab5ef4ff849fef6837561852 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:51:59 -0700 Subject: [PATCH 60/88] Update extra_transition_domain_helper.py --- .../commands/utility/extra_transition_domain_helper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 552997f10..ce8eff5b3 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -615,8 +615,8 @@ class LoadExtraTransitionDomain: # TODO - change name @dataclass -class PatternMap: - """Helper class that holds data and metadata about a requested file. +class FileDataHolder: + """Helper class that holds data about a requested file. filename: str -> The desired filename to target. If no filename is given, it is assumed that you are passing in a filename pattern and it will look @@ -806,7 +806,7 @@ class ExtraTransitionDomain: a list of values. return example: - EnumFilenames.AUTHORITY_ADHOC: PatternMap( + EnumFilenames.AUTHORITY_ADHOC: FileDataHolder( authority_adhoc_filename, self.strip_date_regex, AuthorityAdhoc, @@ -815,7 +815,7 @@ class ExtraTransitionDomain: """ file_data = {} for file_type, filename, data_type, id_field in pattern_map_params: - file_data[file_type] = PatternMap( + file_data[file_type] = FileDataHolder( filename, self.strip_date_regex, data_type, @@ -879,7 +879,7 @@ class ExtraTransitionDomain: def clear_file_data(self): for item in self.file_data.values(): - file_type: PatternMap = item + file_type: FileDataHolder = item file_type.data = {} def parse_csv_file( From a19f2dde3ad8e6d5b0ec8ae1600bab3ec6b1d2b0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 8 Nov 2023 13:30:44 -0600 Subject: [PATCH 61/88] Logging updates --- .../commands/load_transition_domain.py | 35 +++++++++++++------ .../commands/master_domain_migrations.py | 4 +-- .../utility/extra_transition_domain_helper.py | 23 ++++++++++-- .../commands/utility/terminal_helper.py | 16 ++++++++- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 1744807bb..f46994981 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -273,7 +273,7 @@ class Command(BaseCommand): -------------------------------------------- Found {total_outlier_statuses} unaccounted - for statuses- + for statuses -------------------------------------------- No mappings found for the following statuses @@ -601,15 +601,6 @@ class Command(BaseCommand): TransitionDomain.objects.bulk_create(to_create) - logger.info( - f"""{TerminalColors.OKGREEN} - ============= FINISHED =============== - Created {total_new_entries} transition domain entries, - updated {total_updated_domain_entries} transition domain entries - {TerminalColors.ENDC} - """ - ) - # Print a summary of findings (duplicate entries, # missing data..etc.) self.print_summary_duplications( @@ -617,10 +608,32 @@ class Command(BaseCommand): ) self.print_summary_status_findings(domains_without_status, outlier_statuses) + + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED =============== + Created {total_new_entries} transition domain entries, + Updated {total_updated_domain_entries} transition domain entries + + {TerminalColors.YELLOW} + ----- DUPLICATES FOUND ----- + {len(duplicate_domain_user_combos)} DOMAIN - USER pairs + were NOT unique in the supplied data files. + {len(duplicate_domains)} DOMAINS were NOT unique in + the supplied data files. + + ----- STATUSES ----- + {len(domains_without_status)} DOMAINS had NO status (defaulted to READY). + {len(outlier_statuses)} Statuses were invalid (defaulted to READY). + + {TerminalColors.ENDC} + """ + ) + # Prompt the user if they want to load additional data on the domains title = "Do you wish to load additional data for TransitionDomains?" proceed = TerminalHelper.prompt_for_execution( - system_exit_on_terminate=False, + system_exit_on_terminate=True, info_to_inspect=f""" !!! ENSURE THAT ALL FILENAMES ARE CORRECT BEFORE PROCEEDING ==Master data file== diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 27fdcf624..113e1bcf8 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -309,7 +309,7 @@ class Command(BaseCommand): proceed = False if prompts_enabled: proceed = TerminalHelper.prompt_for_execution( - False, + True, command_string, "Running load_transition_domain script", ) @@ -337,7 +337,7 @@ class Command(BaseCommand): proceed = False if prompts_enabled: proceed = TerminalHelper.prompt_for_execution( - False, + True, command_string, "Running transfer_transition_domains_to_domains script", ) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 552997f10..f4d64e615 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -8,6 +8,7 @@ import re import logging import os +import sys from typing import List, Tuple from registrar.models.transition_domain import TransitionDomain @@ -232,9 +233,25 @@ class LoadExtraTransitionDomain: {TerminalColors.ENDC} """ ) - # TODO - if TransitionDomain.objects.all().count() != len(updated_transition_domains): - logger.error("Something bad happened") + + # DATA INTEGRITY CHECK + # Make sure every Transition Domain got updated + total_transition_domains = TransitionDomain.objects.all().count() + total_updates_made = TransitionDomain.objects.all().count() + if total_transition_domains != total_updates_made: + logger.error(f"""{TerminalColors.Fail} + WARNING: something went wrong processing domain information data. + + Total Transition Domains expecting a data update: {total_transition_domains} + Total updates made: {total_updates_made} + + ^ These totals should match, but they don't. This + error should never occur, but could indicate + corrupt data. Please check logs to diagnose. + + ----- TERMINATING ---- + """) + sys.exit() def parse_creation_expiration_data(self, domain_name, transition_domain): """Grabs expiration_date from the parsed files and associates it diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 4f937e699..0f309f9c8 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -185,9 +185,23 @@ class TerminalHelper: li = file.readlines() total_line = len(li) return total_line - + @staticmethod + def print_to_file_conditional(print_condition: bool, filename: str, file_directory: str, file_contents: str): + """Sometimes logger outputs get insanely huge. + """ + if (print_condition): + # Add a slash if the last character isn't one + if file_directory and file_directory[-1] != "/": + file_directory += "/" + # Assemble filepath + filepath = f"{file_directory}{filename}.txt" + # Write to file + logger.info(f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}") + with open(f"{filepath}", "w+") as f: + f.write(file_contents) + @staticmethod def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"): """ From ea5df7adeed66df066123c979d35dcdf5d11b0e5 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 8 Nov 2023 14:37:17 -0600 Subject: [PATCH 62/88] Changed "--prompt" to "--disablePrompts". Updates to documentation. Needs testing --- docs/operations/data_migration.md | 121 ++++++++++++++---- .../commands/load_transition_domain.py | 3 - .../commands/master_domain_migrations.py | 94 ++++---------- .../test_transition_domain_migrations.py | 1 + 4 files changed, 123 insertions(+), 96 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 75bd50a9b..213e7f6ee 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -79,11 +79,36 @@ docker compose run app ./manage.py load_domain_invitations /app/escrow_domain_co ``` ## Transition Domains (Part 1) - Setup Files for Import -We are provided with information about Transition Domains in 3 files: + +#### STEP 1: obtain data files +We are provided with information about Transition Domains in the following files: - FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain names to contact ID. Domains in this file will usually have 3 contacts each - FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) - FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses +- FILE 4: **escrow_domains.daily.dotgov.GOV.txt** -> has basic domain data +- FILE 5: **domainadditionaldatalink.adhoc.dotgov.txt** -> has the map of domains to other data like authority, organization, & domain type +- FILE 6: **domaintypes.adhoc.dotgov.txt** -> has domain type data +- FILE 7: **organization.adhoc.dotgov.txt** -> has organization data +- FILE 8: **authority.adhoc.dotgov.txt** -> has authority data +- FILE 9: **agency.adhoc.dotgov.txt** -> has agency data + +#### STEP 2: obtain JSON file (for file locations) +Add a JSON file called "migrationFilepaths.json" with the following contents (update filenames and directory as needed): +``` +{ + "directory": "migrationdata", + "contacts_filename": "escrow_contacts.daily.dotgov.GOV.txt", + "domain_contacts_filename": "escrow_domain_contacts.daily.dotgov.GOV.txt", + "domain_statuses_filename": "escrow_domain_statuses.daily.dotgov.GOV.txt", + "domain_escrow_filename": "escrow_domains.daily.dotgov.GOV.txt", + "domain_additional_filename": "domainadditionaldatalink.adhoc.dotgov.txt", + "domain_adhoc_filename": "domaintypes.adhoc.dotgov.txt", + "organization_adhoc_filename": "organization.adhoc.dotgov.txt" + "authority_adhoc_filename": "authority.adhoc.dotgov.txt", + "agency_adhoc_filename": "agency.adhoc.dotgov.txt", +} +``` We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. @@ -203,7 +228,7 @@ cat ../tmp/{filename} > migrationdata/{filename} In order to run the scripts locally, we need to add the files to a folder under `src/`. This will allow Docker to mount the files to a container (under `/app`) for our use. - - Add the above files to the `migrationdata/` folder + - Add the same files from section 1 to a TEMPORARY `tmp/` folder under `src/` (do not check this folder into our repo) - Open a terminal and navigate to `src/` @@ -216,9 +241,13 @@ This will allow Docker to mount the files to a container (under `/app`) for our Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. -(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and and remove "docker compose run -T app" from the command) +##### LOCAL COMMAND ```shell -docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug +docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp --debug +``` +##### SANDBOX COMMAND +```shell +./manage.py load_transition_domain migrationFilepaths.json --debug ``` ##### COMMAND LINE ARGUMENTS: @@ -232,6 +261,41 @@ Directs the script to load only the first 100 entries into the table. You can a `--resetTable` This will delete all the data in transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. +###### (arguments that override filepaths and directories if needed) +`--infer_filenames` +Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.. + +`--directory` +Defines the directory where all data files and the JSON are stored. + +`--domain_contacts_filename` +Defines the filename for domain contact information. + +`--contacts_filename` +Defines the filename for contact information. + +`--domain_statuses_filename` +Defines the filename for domain status information. + +`--agency_adhoc_filename` +Defines the filename for agency adhocs. + +`--domain_additional_filename` +Defines the filename for additional domain data. + +`--domain_escrow_filename` +Defines the filename for creation/expiration domain data. + +`--domain_adhoc_filename` +Defines the filename for domain type adhocs. + +`--organization_adhoc_filename` +Defines the filename for domain type adhocs. + +`--authority_adhoc_filename` +Defines the filename for domain type adhocs. + + ### STEP 2: Transfer Transition Domain data into main Domain tables @@ -239,10 +303,14 @@ Now that we've loaded all the data into TransitionDomain, we need to update the In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) +##### LOCAL COMMAND ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` +##### SANDBOX COMMAND +```shell +./manage.py transfer_transition_domains_to_domains --debug +``` ##### COMMAND LINE ARGUMENTS: @@ -256,10 +324,14 @@ Directs the script to load only the first 100 entries into the table. You can a To send invitation emails for every transition domain in the transition domain table, execute the following command: -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) +##### LOCAL COMMAND ```shell docker compose run -T app ./manage.py send_domain_invitations -s ``` +##### SANDBOX COMMAND +```shell +./manage.py send_domain_invitations -s +``` ### STEP 4: Test the results (Run the analyzer script) @@ -269,18 +341,27 @@ This script's main function is to scan the transition domain and domain tables f To analyze our database without running migrations, execute the script without any optional arguments: -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) +##### LOCAL COMMAND ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` +##### SANDBOX COMMAND +```shell +./manage.py master_domain_migrations --debug +``` #### OPTION 2 - RUN MIGRATIONS FEATURE -To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: +To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug provides detailed logging statements during the migration. It is recommended that you use this argument when using the --runMigrations feature: -(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) +(NOTE: If you named your JSON file something other than "migrationFilepaths.json" (all the way back in the "file setup" section). You will want to utilize the `--migrationJSON` argument in the following commands...) +##### LOCAL COMMAND ```shell -docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt +docker compose run -T app ./manage.py master_domain_migrations --migrationDirectory /app/tmp --runMigrations --debug +``` +##### SANDBOX COMMAND +```shell +./manage.py master_domain_migrations --runMigrations --debug ``` ##### COMMAND LINE ARGUMENTS @@ -291,25 +372,18 @@ Runs all scripts (in sequence) for transition domain migrations `--migrationDirectory` -The location of the files used for load_transition_domain migration script. +The location of both the JSON file and all files needed for migration. (default is "migrationdata" (This is the sandbox directory)) Example Usage: *--migrationDirectory /app/tmp* -`--migrationFilenames` +`--migrationJSON` -The filenames used for load_transition_domain migration script. -Must appear *in oprder* and comma-delimited: -default is "escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt" -where... -- domain_contacts_filename is the Data file with domain contact information -- contacts_filename is the Data file with contact information -- domain_statuses_filename is the Data file with domain status information +The filename of the JSON that holds all the filepath info needed for migrations. Example Usage: -*--migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt* - +*--migrationJSON migrationFilepaths.json* `--sep` @@ -320,9 +394,10 @@ Delimiter for the migration scripts to correctly parse the given text files. Activates additional print statements -`--prompt` +`--disablePrompts` -Activates terminal prompts that allows the user to step through each portion of this script. +Disables the terminal prompts that allows the user to step through each portion of this script. +*used to facilitate unit tests. Not recommended for everyday use* `--limitParse` diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index f46994981..df1974a18 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -63,9 +63,6 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - # TODO - Narrow this down - # TODO - this isn't pulling in the directory from the master script. Needs to be corrected @Nicolle - todo - # default="/app/tmp" parser.add_argument( "--directory", default="migrationdata", help="Desired directory" ) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 113e1bcf8..c3f92a3a9 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -45,27 +45,21 @@ class Command(BaseCommand): The location of the files used for load_transition_domain migration script EXAMPLE USAGE: > --migrationDirectory /app/tmp - - --migrationFilenames - The files used for load_transition_domain migration script. - Must appear IN ORDER and comma-delimiteds: + + --migrationJSON + The name of the JSON file used for load_transition_domain migration script EXAMPLE USAGE: - > --migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt - where... - - domain_contacts_filename is the Data file with domain contact information - - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information + > --migrationJSON migrationFilepaths.json --sep Delimiter for the migration scripts to correctly parse the given text files. (usually this can remain at default value of |) --debug - A boolean (default to true), which activates additional print statements + Activates additional print statements - --prompt - A boolean (default to true), which activates terminal prompts - that allows the user to step through each portion of this + --disablePrompts + Disables terminal prompts that allows the user to step through each portion of this script. --limitParse @@ -99,7 +93,8 @@ class Command(BaseCommand): # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( - "migration_json_filename", + "--migrationJSON", + default="migrationFilepaths.json", help=( "A JSON file that holds the location and filenames" "of all the data files used for migrations" @@ -116,31 +111,13 @@ class Command(BaseCommand): ), ) - # TODO: deprecate this once JSON module is done? (or keep as an override) - parser.add_argument( - "--migrationFilenames", - default="escrow_domain_contacts.daily.gov.GOV.txt," - "escrow_contacts.daily.gov.GOV.txt," - "escrow_domain_statuses.daily.gov.GOV.txt", - help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by commas: - domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt - - where... - - domain_contacts_filename is the Data file with domain contact - information - - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status - information""", - ) - parser.add_argument( "--sep", default="|", help="Delimiter character for the migration files" ) parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) + parser.add_argument("--disablePrompts", action=argparse.BooleanOptionalAction) parser.add_argument( "--limitParse", default=0, help="Sets max number of entries to load" @@ -282,19 +259,23 @@ class Command(BaseCommand): def run_load_transition_domain_script( self, migration_json_filename: str, - file_location: str, + file_directory: str, sep: str, reset_table: bool, debug_on: bool, prompts_enabled: bool, debug_max_entries_to_parse: int, ): + + if file_directory and file_directory[-1] != "/": + file_directory += "/" + json_filepath = file_directory+migration_json_filename """Runs the load_transition_domain script""" # Create the command string command_script = "load_transition_domain" command_string = ( f"./manage.py {command_script} " - f"{file_location+migration_json_filename} " + f"{json_filepath} " ) if sep is not None and sep != "|": command_string += f"--sep {sep} " @@ -318,12 +299,12 @@ class Command(BaseCommand): if proceed or not prompts_enabled: call_command( command_script, - f"{file_location+migration_json_filename}", + json_filepath, sep=sep, resetTable=reset_table, debug=debug_on, limitParse=debug_max_entries_to_parse, - directory=file_location + directory=file_directory ) def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): @@ -367,9 +348,6 @@ class Command(BaseCommand): self, migration_json_filename, file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, sep, reset_table, debug_on, @@ -392,10 +370,8 @@ class Command(BaseCommand): The migration scripts are looking in directory.... {file_location} - ....for the following files: - - domain contacts: {domain_contacts_filename} - - contacts: {contacts_filename} - - domain statuses: {domain_statuses_filename} + ....for the following JSON: + {migration_json_filename} {TerminalColors.FAIL} Does this look correct?{TerminalColors.ENDC}""" @@ -410,7 +386,7 @@ class Command(BaseCommand): f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct - file location and filenames: + JSON filename and directory: """ ) return @@ -429,7 +405,6 @@ class Command(BaseCommand): def handle( self, - migration_json_filename, **options, ): """ @@ -452,7 +427,7 @@ class Command(BaseCommand): # Get arguments debug_on = options.get("debug") - prompts_enabled = options.get("prompt") + prompts_enabled = not options.get("disablePrompts") run_migrations_enabled = options.get("runMigrations") TerminalHelper.print_conditional( @@ -501,34 +476,13 @@ class Command(BaseCommand): debug_max_entries_to_parse = int(options.get("limitParse")) # Grab filepath information from the arguments - file_location = options.get("migrationDirectory") + "/" - filenames = options.get("migrationFilenames").split(",") - if len(filenames) < 3: - filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info( - f""" - {TerminalColors.FAIL} - --migrationFilenames expected 3 filenames to follow it, - but only {len(filenames)} were given: - {filenames_as_string} - - PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN - ============= TERMINATING ============= - {TerminalColors.ENDC} - """ - ) - sys.exit() - domain_contacts_filename = filenames[0] - contacts_filename = filenames[1] - domain_statuses_filename = filenames[2] + file_location = options.get("migrationDirectory") + migration_json_filename = options.get("migrationJSON") # Run migration scripts self.run_migration_scripts( migration_json_filename, file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, sep, reset_table, debug_on, diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 7781e66e9..d0cfef8c9 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -63,6 +63,7 @@ class TestMigrations(TestCase): runMigrations=True, migrationDirectory=self.test_data_file_location, migration_json_filename=self.migration_json_filename, + disablePrompts=True ) def compare_tables( From 2a7d26dff9609a0ad60d3706ca321d99374fd023 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:38:21 -0700 Subject: [PATCH 63/88] Test changes --- .../utility/extra_transition_domain_helper.py | 2 +- .../test_transition_domain_migrations.py | 82 +++++++++++++------ 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index ce8eff5b3..0414aac00 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -609,7 +609,7 @@ class LoadExtraTransitionDomain: obj = desired_type.data.get(desired_id) if obj is None: self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Id {desired_id} does not exist" + file_type, LogCode.ERROR, f"Id {desired_id} does not exist for {file_type.value[0]}" ) return obj diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 7781e66e9..a6eff207f 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -277,7 +277,7 @@ class TestMigrations(TestCase): # Each TransitionDomain should have the correct data self.assertEqual(domain, expected) - def test_load_full_transfer_domain(self): + def test_load_full_domain(self): self.run_load_domains() self.run_transfer_domains() @@ -302,47 +302,79 @@ class TestMigrations(TestCase): expected_missing_domain_invitations, ) + # Test created domains anomaly_domains = Domain.objects.filter(name="anomaly.gov") self.assertEqual(anomaly_domains.count(), 1) anomaly = anomaly_domains.get() self.assertEqual(anomaly.expiration_date, datetime.date(2023, 3, 9)) + """ self.assertEqual( anomaly.created_at, datetime.datetime(2023, 11, 8, 17, 23, 46, 764663, tzinfo=datetime.timezone.utc) ) + """ self.assertEqual(anomaly.name, "anomaly.gov") self.assertEqual(anomaly.state, "ready") - testdomain_domains = Domain.objects.filter(name="testdomain.gov") + testdomain_domains = Domain.objects.filter(name="fakewebsite2.gov") self.assertEqual(testdomain_domains.count(), 1) testdomain = testdomain_domains.get() - self.assertEqual(testdomain.expiration_date, datetime.date(2023, 3, 9)) - self.assertEqual(testdomain.created_at, "test") - self.assertEqual(testdomain.name, "anomaly.gov") - self.assertEqual(testdomain.state, "ready") + self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) + #self.assertEqual(testdomain.created_at, "test") + self.assertEqual(testdomain.name, "fakewebsite2.gov") + self.assertEqual(testdomain.state, "on hold") + + def test_load_full_domain_information(self): + self.run_load_domains() + self.run_transfer_domains() - expected_domains = [ - Domain( - expiration_date=None, - name="anomaly.gov", - state="ready", - ), - Domain( - expiration_date=None, - name="testdomain.gov", - state="ready", - ), - ] - - for domain in Domain.objects.all(): - for expected in expected_domains: - expected.id = domain.id - expected.created_at = domain.created_at - expected.updated_at = domain.updated_at + # Analyze the tables + expected_total_transition_domains = 9 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 - self.assertEqual(domain, expected) + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 0 + expected_missing_domain_invitations = 1 + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + # Test created Domain Information objects + domain = Domain.objects.filter(name="anomaly.gov").get() + anomaly_domain_infos = DomainInformation.objects.filter(domain=domain) + self.assertEqual(anomaly_domain_infos.count(), 1) + anomaly = anomaly_domain_infos.get() + + self.assertEqual(anomaly.expiration_date, datetime.date(2023, 3, 9)) + """ + self.assertEqual( + anomaly.created_at, datetime.datetime(2023, 11, 8, 17, 23, 46, 764663, tzinfo=datetime.timezone.utc) + ) + """ + self.assertEqual(anomaly.name, "anomaly.gov") + self.assertEqual(anomaly.state, "ready") + + testdomain_domains = Domain.objects.filter(name="fakewebsite2.gov") + self.assertEqual(testdomain_domains.count(), 1) + + testdomain = testdomain_domains.get() + + self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) + #self.assertEqual(testdomain.created_at, "test") + self.assertEqual(testdomain.name, "fakewebsite2.gov") + self.assertEqual(testdomain.state, "on hold") def test_transfer_transition_domains_to_domains(self): self.run_load_domains() From a32361146ef19d73d84793b143aa9858702f53ce Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:22:32 -0700 Subject: [PATCH 64/88] Update test_transition_domain_migrations.py --- .../test_transition_domain_migrations.py | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index fdc5af5e0..841f847d8 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -355,27 +355,37 @@ class TestMigrations(TestCase): # Test created Domain Information objects domain = Domain.objects.filter(name="anomaly.gov").get() anomaly_domain_infos = DomainInformation.objects.filter(domain=domain) + self.assertEqual(anomaly_domain_infos.count(), 1) + + # This domain should be pretty barebones. Something isnt + # parsing right if we get a lot of data. anomaly = anomaly_domain_infos.get() + self.assertEqual(anomaly.organization_name, "Flashdog") + self.assertEqual(anomaly.organization_type, None) + self.assertEqual(anomaly.federal_agency, None) + self.assertEqual(anomaly.federal_type, None) + + # Check for the "system" creator user + Users = User.objects.filter(username="System") + self.assertEqual(Users.count(), 1) + self.assertEqual(anomaly.creator, Users.get()) - self.assertEqual(anomaly.expiration_date, datetime.date(2023, 3, 9)) - """ - self.assertEqual( - anomaly.created_at, datetime.datetime(2023, 11, 8, 17, 23, 46, 764663, tzinfo=datetime.timezone.utc) - ) - """ - self.assertEqual(anomaly.name, "anomaly.gov") - self.assertEqual(anomaly.state, "ready") + domain = Domain.objects.filter(name="fakewebsite2.gov") + fakewebsite_domain_infos = DomainInformation.objects.filter(domain=domain) + self.assertEqual(fakewebsite_domain_infos.count(), 1) - testdomain_domains = Domain.objects.filter(name="fakewebsite2.gov") - self.assertEqual(testdomain_domains.count(), 1) + fakewebsite = fakewebsite_domain_infos.get() + self.assertEqual(fakewebsite.organization_name, "Flashdog") + self.assertEqual(fakewebsite.organization_type, None) + self.assertEqual(fakewebsite.federal_agency, None) + self.assertEqual(fakewebsite.federal_type, None) + + # Check for the "system" creator user + Users = User.objects.filter(username="System") + self.assertEqual(Users.count(), 1) + self.assertEqual(anomaly.creator, Users.get()) - testdomain = testdomain_domains.get() - - self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) - #self.assertEqual(testdomain.created_at, "test") - self.assertEqual(testdomain.name, "fakewebsite2.gov") - self.assertEqual(testdomain.state, "on hold") def test_transfer_transition_domains_to_domains(self): self.run_load_domains() From 50f6b49b47081b76cbb3bd85294b427ccd3e166d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:44:30 -0700 Subject: [PATCH 65/88] Update settings.py --- src/registrar/config/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 87ad0ff96..2e061c1a9 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -72,6 +72,8 @@ secret_registry_hostname = secret("REGISTRY_HOSTNAME") BASE_DIR = path.resolve().parent.parent.parent # SECURITY WARNING: don't run with debug turned on in production! +# TODO - Investigate the behaviour of this flag. Does not appear +# to function for the IS_PRODUCTION flag. DEBUG = env_debug # Controls production specific feature toggles From 638493a6e21c36e47f329d285b3fb9646e17051a Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 8 Nov 2023 16:08:34 -0600 Subject: [PATCH 66/88] documentation updates and updates to json arg --- docs/operations/data_migration.md | 2 +- .../commands/load_transition_domain.py | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 213e7f6ee..c4d2c150c 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -243,7 +243,7 @@ Run the following command, making sure the file paths point to the right locatio ##### LOCAL COMMAND ```shell -docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp --debug +docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp/ --debug --limitParse 10 ``` ##### SANDBOX COMMAND ```shell diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index df1974a18..a38a7366e 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -64,9 +64,14 @@ class Command(BaseCommand): ) parser.add_argument( - "--directory", default="migrationdata", help="Desired directory" + "--infer_filenames", + action=argparse.BooleanOptionalAction, + help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." ) + parser.add_argument( + "--directory", default="migrationdata", help="Desired directory" + ) parser.add_argument( "--domain_contacts_filename", help="Data file with domain contact information" @@ -79,11 +84,6 @@ class Command(BaseCommand): "--domain_statuses_filename", help="Data file with domain status information" ) - parser.add_argument( - "--infer_filenames", - action=argparse.BooleanOptionalAction, - help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." - ) parser.add_argument( "--agency_adhoc_filename", default=EnumFilenames.AGENCY_ADHOC.value[1], @@ -322,10 +322,19 @@ class Command(BaseCommand): ): """Parse the data files and create TransitionDomains.""" args = TransitionDomainArguments(**options) + + # Desired directory for additional TransitionDomain data + # (In the event they are stored seperately) + directory = args.directory + # Add a slash if the last character isn't one + if directory and directory[-1] != "/": + directory += "/" + + json_filepath = directory + migration_json_filename ### Process JSON file ### # If a JSON was provided, use its values instead of defaults. # TODO: there is no way to discern user overrides from those arg’s defaults. - with open(migration_json_filename, "r") as jsonFile: + with open(json_filepath, "r") as jsonFile: # load JSON object as a dictionary try: data = json.load(jsonFile) @@ -361,12 +370,6 @@ class Command(BaseCommand): ## Variables for Additional TransitionDomain Information ## - # Desired directory for additional TransitionDomain data - # (In the event they are stored seperately) - directory = args.directory - # Add a slash if the last character isn't one - if directory and directory[-1] != "/": - directory += "/" # Main script filenames - these do not have defaults domain_contacts_filename = None From c28c2e634253ee8237cf477d4025c5f8a95d958c Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 8 Nov 2023 16:36:04 -0600 Subject: [PATCH 67/88] Updated logger statements --- .../transfer_transition_domains_to_domains.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index e58d7dc24..a4e81abf5 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -634,18 +634,16 @@ class Command(BaseCommand): break - logger.info( - f"""{TerminalColors.OKCYAN} - ========= Adding Domains Information Objects ========= - {TerminalColors.ENDC}""" - ) - # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) #DomainInvitation.objects.bulk_create(domain_invitations_to_create) + # TODO: this is to resolve an error where bulk_create + # doesn't save to database in a way that invitation objects can + # utilize. # Then, create DomainInvitation objects for invitation in domain_invitations_to_create: + logger.info(f"Pairing invite to its domain...{invitation}") existing_domain = Domain.objects.filter(name=invitation.domain.name) # Make sure the related Domain object is saved if existing_domain.exists(): @@ -657,6 +655,11 @@ class Command(BaseCommand): # ====================================================== # ================= DOMAIN INFORMATION ================= + logger.info( + f"""{TerminalColors.OKCYAN} + ========= Adding Domains Information Objects ========= + {TerminalColors.ENDC}""" + ) for transition_domain in TransitionDomain.objects.all(): target_domain_information, associated_domain, was_created = self.update_or_create_domain_information(transition_domain, debug_on) From 316516710ba6c7d547850751f1e3ccfec9179887 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:48:34 -0700 Subject: [PATCH 68/88] Add test cases --- .../tests/data/test_agency_adhoc.txt | 4 +-- .../test_transition_domain_migrations.py | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/registrar/tests/data/test_agency_adhoc.txt b/src/registrar/tests/data/test_agency_adhoc.txt index 0ff33b852..85213aa97 100644 --- a/src/registrar/tests/data/test_agency_adhoc.txt +++ b/src/registrar/tests/data/test_agency_adhoc.txt @@ -2,5 +2,5 @@ agencyid|agencyname|active|isfederal 1|Thoughtstorm|N|Y 2|Minyx|Y|N 3|Demivee|N|Y -4|InnoZ|Y|Y -5|igorville|Y|N \ No newline at end of file +4|Department of Commerce|Y|Y +5|InnoZ|Y|Y diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 841f847d8..3ac2fa2e5 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -49,7 +49,7 @@ class TestMigrations(TestCase): with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): call_command( "load_transition_domain", - f"{self.test_data_file_location}/{self.migration_json_filename}", + self.migration_json_filename, directory=self.test_data_file_location ) @@ -62,7 +62,7 @@ class TestMigrations(TestCase): "master_domain_migrations", runMigrations=True, migrationDirectory=self.test_data_file_location, - migration_json_filename=self.migration_json_filename, + migrationJSON=self.migration_json_filename, disablePrompts=True ) @@ -190,10 +190,10 @@ class TestMigrations(TestCase): expected_total_domain_informations = 0 expected_total_domain_invitations = 0 - expected_missing_domains = 8 + expected_missing_domains = 9 expected_duplicate_domains = 0 - expected_missing_domain_informations = 8 - expected_missing_domain_invitations = 8 + expected_missing_domain_informations = 9 + expected_missing_domain_invitations = 9 self.compare_tables( expected_total_transition_domains, expected_total_domains, @@ -371,15 +371,15 @@ class TestMigrations(TestCase): self.assertEqual(Users.count(), 1) self.assertEqual(anomaly.creator, Users.get()) - domain = Domain.objects.filter(name="fakewebsite2.gov") + domain = Domain.objects.filter(name="fakewebsite2.gov").get() fakewebsite_domain_infos = DomainInformation.objects.filter(domain=domain) self.assertEqual(fakewebsite_domain_infos.count(), 1) fakewebsite = fakewebsite_domain_infos.get() - self.assertEqual(fakewebsite.organization_name, "Flashdog") - self.assertEqual(fakewebsite.organization_type, None) - self.assertEqual(fakewebsite.federal_agency, None) - self.assertEqual(fakewebsite.federal_type, None) + self.assertEqual(fakewebsite.organization_name, "Fanoodle") + self.assertEqual(fakewebsite.organization_type, "federal") + self.assertEqual(fakewebsite.federal_agency, "Department of Commerce") + self.assertEqual(fakewebsite.federal_type, "executive") # Check for the "system" creator user Users = User.objects.filter(username="System") @@ -393,9 +393,9 @@ class TestMigrations(TestCase): # Analyze the tables expected_total_transition_domains = 9 - expected_total_domains = 4 - expected_total_domain_informations = 4 - expected_total_domain_invitations = 7 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 expected_missing_domains = 0 expected_duplicate_domains = 0 @@ -426,10 +426,10 @@ class TestMigrations(TestCase): user.first_login() # Analyze the tables - expected_total_transition_domains = 8 - expected_total_domains = 4 - expected_total_domain_informations = 4 - expected_total_domain_invitations = 7 + expected_total_transition_domains = 9 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 expected_missing_domains = 0 expected_duplicate_domains = 0 From 9409b3f41cdc52fa0f0ac98fba5ceaa07375cfe2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:32:02 -0700 Subject: [PATCH 69/88] Fix test cases + logging fixes --- .../commands/load_transition_domain.py | 2 +- .../commands/master_domain_migrations.py | 4 +++- .../utility/extra_transition_domain_helper.py | 22 +++++++++---------- src/registrar/models/transition_domain.py | 4 ++++ .../test_transition_domain_migrations.py | 4 ++-- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index a38a7366e..3878a3366 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -339,7 +339,7 @@ class Command(BaseCommand): try: data = json.load(jsonFile) # Create an instance of TransitionDomainArguments - + has_desired_args = False # Iterate over the data from the JSON file for key, value in data.items(): # Check if the key exists in TransitionDomainArguments diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index c3f92a3a9..96bf0517b 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -269,7 +269,7 @@ class Command(BaseCommand): if file_directory and file_directory[-1] != "/": file_directory += "/" - json_filepath = file_directory+migration_json_filename + json_filepath = migration_json_filename """Runs the load_transition_domain script""" # Create the command string command_script = "load_transition_domain" @@ -285,6 +285,8 @@ class Command(BaseCommand): command_string += "--debug " if debug_max_entries_to_parse > 0: command_string += f"--limitParse {debug_max_entries_to_parse} " + if file_directory: + command_string += f"--directory {file_directory}" # Execute the command string proceed = False diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 06879c515..fd7752839 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -185,11 +185,8 @@ class LoadExtraTransitionDomain: ) # Check if the instance has changed before saving - #if updated_transition_domain.__dict__ != transition_domain.__dict__: - updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) - logger.info( f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" @@ -208,17 +205,16 @@ class LoadExtraTransitionDomain: ) failed_transition_domains.append(domain_name) - if self.debug: - # Display misc errors (not associated to a domain) - self.parse_logs.display_logs_by_domain_name(None) - failed_count = len(failed_transition_domains) if failed_count == 0: - TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") + if self.debug: + for domain in updated_transition_domains: + logger.debug(domain.display_transition_domain()) logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Updated {len(updated_transition_domains)} transition domain entries: + {[domain for domain in updated_transition_domains]} {TerminalColors.ENDC} """ ) @@ -228,18 +224,20 @@ class LoadExtraTransitionDomain: logger.error( f"""{TerminalColors.FAIL} ============= FINISHED WITH ERRORS =============== - Updated {len(updated_transition_domains)} transition domain entries, - Failed to update {failed_count} transition domain entries + Updated {len(updated_transition_domains)} transition domain entries: + {[domain for domain in updated_transition_domains]} + Failed to update {failed_count} transition domain entries: + {[domain for domain in failed_transition_domains]} {TerminalColors.ENDC} """ ) # DATA INTEGRITY CHECK # Make sure every Transition Domain got updated - total_transition_domains = TransitionDomain.objects.all().count() + total_transition_domains = len(updated_transition_domains) total_updates_made = TransitionDomain.objects.all().count() if total_transition_domains != total_updates_made: - logger.error(f"""{TerminalColors.Fail} + logger.error(f"""{TerminalColors.FAIL} WARNING: something went wrong processing domain information data. Total Transition Domains expecting a data update: {total_transition_domains} diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index a31b27e7e..7c4d2afe2 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -79,6 +79,10 @@ class TransitionDomain(TimeStampedModel): ) def __str__(self): + return f"{self.username}, {self.domain_name}" + + def display_transition_domain(self): + """Displays all information about a TransitionDomain in string format""" return ( f"\n-----TRANSITION DOMAIN------\n" f"domainName: {self.domain_name}, \n" diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 3ac2fa2e5..f5ffb4600 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -159,8 +159,8 @@ class TestMigrations(TestCase): # STEP 2: (analyze the tables just like the # migration script does, but add assert statements) expected_total_transition_domains = 9 - expected_total_domains = 4 - expected_total_domain_informations = 4 + expected_total_domains = 5 + expected_total_domain_informations = 5 expected_total_domain_invitations = 7 expected_missing_domains = 0 From 7d798e0e430235a2afff0318e923e70c930e0f7d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:32:58 -0700 Subject: [PATCH 70/88] Update test_transition_domain_migrations.py --- src/registrar/tests/test_transition_domain_migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index f5ffb4600..877737f43 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -161,7 +161,7 @@ class TestMigrations(TestCase): expected_total_transition_domains = 9 expected_total_domains = 5 expected_total_domain_informations = 5 - expected_total_domain_invitations = 7 + expected_total_domain_invitations = 8 expected_missing_domains = 0 expected_duplicate_domains = 0 From 72d95f6fad9df8cebe5abc22673450a6e71fc0c6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:37:59 -0700 Subject: [PATCH 71/88] Linting + minimize logging to a degree --- docs/operations/data_migration.md | 18 +- .../commands/agency_data_extractor.py | 181 +++++++----- .../commands/load_transition_domain.py | 37 ++- .../commands/master_domain_migrations.py | 17 +- .../transfer_transition_domains_to_domains.py | 270 +++++++++++------- .../commands/utility/epp_data_containers.py | 91 +++--- .../utility/extra_transition_domain_helper.py | 126 ++++---- .../commands/utility/terminal_helper.py | 64 +++-- .../utility/transition_domain_arguments.py | 31 +- .../test_transition_domain_migrations.py | 43 +-- src/registrar/views/utility/mixins.py | 2 +- 11 files changed, 507 insertions(+), 373 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index c4d2c150c..a77266282 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -240,7 +240,23 @@ This will allow Docker to mount the files to a container (under `/app`) for our ### STEP 1: Load Transition Domains Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. - +##### Create a JSON file +In your chosen directory (either `src/tmp` or `src/migrationdata` depending on preference), create a json file called `migrationFilepaths.json`. This file will map to other urls +Example +``` +{ + "directory": "migrationdata/", + "agency_adhoc_filename": "20231009.agency.adhoc.dotgov.txt", + "authority_adhoc_filename": "authority.adhoc.dotgov.txt", + "contacts_filename": "escrow_contacts.daily.dotgov.GOV.txt", + "domain_adhoc_filename": "20231009.domaintypes.adhoc.dotgov.txt", + "domain_additional_filename": "20231009.domainadditionaldatalink.adhoc.dotgov.txt", + "domain_contacts_filename": "escrow_domain_contacts.daily.dotgov.GOV.txt", + "domain_escrow_filename": "escrow_domains.daily.dotgov.GOV.txt", + "domain_statuses_filename": "escrow_domain_statuses.daily.dotgov.GOV.txt", + "organization_adhoc_filename": "20231009.organization.adhoc.dotgov.txt" +} +``` ##### LOCAL COMMAND ```shell docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp/ --debug --limitParse 10 diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py index d5b304a8a..1f0d6e0da 100644 --- a/src/registrar/management/commands/agency_data_extractor.py +++ b/src/registrar/management/commands/agency_data_extractor.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) # Example command for running this script: # docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug + class Command(BaseCommand): help = """Loads data for domains that are in transition (populates transition_domain model objects).""" @@ -26,40 +27,42 @@ class Command(BaseCommand): parser.add_argument( "agency_data_filename", help="Data file with agency information" ) - parser.add_argument( - "--dir", default="migrationdata", help="Desired directory" - ) + parser.add_argument("--dir", default="migrationdata", help="Desired directory") parser.add_argument("--sep", default="|", help="Delimiter character") - parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction) + parser.add_argument( + "--debug", + help="Prints additional debug statements to the terminal", + action=argparse.BooleanOptionalAction, + ) parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) @staticmethod - def extract_agencies( - agency_data_filepath: str, - sep: str, - debug: bool - ) -> [str]: - """Extracts all the agency names from the provided + def extract_agencies(agency_data_filepath: str, sep: str, debug: bool) -> [str]: + """Extracts all the agency names from the provided agency file (skips any duplicates) and returns those names in an array""" agency_names = [] - logger.info(f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}") + logger.info( + f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}" + ) with open(agency_data_filepath, "r") as agency_data_filepath: # noqa for row in csv.reader(agency_data_filepath, delimiter=sep): agency_name = row[1] TerminalHelper.print_conditional(debug, f"Checking: {agency_name}") if agency_name not in agency_names: agency_names.append(agency_name) - logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}") + logger.info( + f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}" + ) return agency_names - + @staticmethod - def compare_agency_lists(provided_agencies: [str], - existing_agencies: [str], - debug: bool): + def compare_agency_lists( + provided_agencies: [str], existing_agencies: [str], debug: bool + ): """ - Compares new_agencies with existing_agencies and + Compares new_agencies with existing_agencies and provides the equivalent of an outer-join on the two (printed to the terminal) """ @@ -69,27 +72,37 @@ class Command(BaseCommand): for agency in provided_agencies: if agency not in existing_agencies and agency not in new_agencies: new_agencies.append(agency) - TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}") + TerminalHelper.print_conditional( + debug, + f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}", + ) possibly_unused_agencies = [] # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) for agency in existing_agencies: - if agency not in provided_agencies and agency not in possibly_unused_agencies: + if ( + agency not in provided_agencies + and agency not in possibly_unused_agencies + ): possibly_unused_agencies.append(agency) - TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") + TerminalHelper.print_conditional( + debug, + f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}", + ) matched_agencies = [] for agency in provided_agencies: if agency in existing_agencies: matched_agencies.append(agency) - TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}") + TerminalHelper.print_conditional( + debug, + f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}", + ) # Print the summary of findings # 1 - Print the list of agencies in the NEW list, which we do not already have # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? - new_agencies_as_string = "{}".format( - ",\n ".join(map(str, new_agencies)) - ) + new_agencies_as_string = "{}".format(",\n ".join(map(str, new_agencies))) possibly_unused_agencies_as_string = "{}".format( ",\n ".join(map(str, possibly_unused_agencies)) ) @@ -97,7 +110,8 @@ class Command(BaseCommand): ",\n ".join(map(str, matched_agencies)) ) - logger.info(f""" + logger.info( + f""" {TerminalColors.OKGREEN} ======================== SUMMARY OF FINDINGS ============================ {len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file. @@ -117,13 +131,12 @@ class Command(BaseCommand): These agencies are in our system, but not in the provided agency file: {TerminalColors.YELLOW}{possibly_unused_agencies_as_string} {TerminalColors.ENDC} - """) - + """ + ) + @staticmethod def print_agency_list(agencies, filename): - full_agency_list_as_string = "{}".format( - ",\n".join(map(str, agencies)) - ) + full_agency_list_as_string = "{}".format(",\n".join(map(str, agencies))) logger.info( f"\n{TerminalColors.YELLOW}" f"\n{full_agency_list_as_string}" @@ -146,19 +159,22 @@ class Command(BaseCommand): prompt = options.get("prompt") dir = options.get("dir") - agency_data_file = dir+"/"+agency_data_filename + agency_data_file = dir + "/" + agency_data_filename new_agencies = self.extract_agencies(agency_data_file, sep, debug) hard_coded_agencies = DomainApplication.AGENCIES - transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct() + transition_domain_agencies = ( + TransitionDomain.objects.all() + .values_list("federal_agency", flat=True) + .distinct() + ) print(transition_domain_agencies) - merged_agencies = new_agencies for agency in hard_coded_agencies: if agency not in merged_agencies: merged_agencies.append(agency) - + merged_transition_agencies = new_agencies for agency in transition_domain_agencies: if agency not in merged_transition_agencies: @@ -168,73 +184,90 @@ class Command(BaseCommand): # OPTION to compare the agency file to our hard-coded list if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: self.compare_agency_lists(new_agencies, hard_coded_agencies, debug) - + # OPTION to compare the agency file to Transition Domains if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: self.compare_agency_lists(new_agencies, transition_domain_agencies, debug) # OPTION to print out the full list of agencies from the agency file if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF IMPORTED AGENCIES ============================" - f"\nThese are all the agencies provided by the given agency file." - f"\n\n{len(new_agencies)} TOTAL\n\n" + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF IMPORTED AGENCIES ============================" + f"\nThese are all the agencies provided by the given agency file." + f"\n\n{len(new_agencies)} TOTAL\n\n" ) self.print_agency_list(new_agencies, "Imported_Agencies") - + # OPTION to print out the full list of agencies from the agency file if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF AGENCIES IN DROPDOWN ============================" - f"\nThese are all the agencies hard-coded in our system for the dropdown list." - f"\n\n{len(hard_coded_agencies)} TOTAL\n\n" + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF AGENCIES IN DROPDOWN ============================" + f"\nThese are all the agencies hard-coded in our system for the dropdown list." + f"\n\n{len(hard_coded_agencies)} TOTAL\n\n" ) self.print_agency_list(hard_coded_agencies, "Dropdown_Agencies") - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF AGENCIES IN TRANSITION DOMAIN ============================" - f"\nThese are all the agencies in the Transition Domains table." - f"\n\n{len(transition_domain_agencies)} TOTAL\n\n" - ) - self.print_agency_list(transition_domain_agencies, "Transition_Domain_Agencies") - # OPTION to print out the full list of agencies from the agency file if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== MERGED LISTS (dropdown + agency file) ============================" - f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." - f"\n\n{len(merged_agencies)} TOTAL\n\n" + f"\n{TerminalColors.OKGREEN}" + f"\n======================== FULL LIST OF AGENCIES IN TRANSITION DOMAIN ============================" + f"\nThese are all the agencies in the Transition Domains table." + f"\n\n{len(transition_domain_agencies)} TOTAL\n\n" + ) + self.print_agency_list( + transition_domain_agencies, "Transition_Domain_Agencies" + ) + + # OPTION to print out the full list of agencies from the agency file + if prompt: + prompt_successful = TerminalHelper.query_yes_no( + f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}" + ) + if prompt_successful or not prompt: + logger.info( + f"\n{TerminalColors.OKGREEN}" + f"\n======================== MERGED LISTS (dropdown + agency file) ============================" + f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." + f"\n\n{len(merged_agencies)} TOTAL\n\n" ) self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") - - # OPTION to print out the full list of agencies from the agency file + + # OPTION to print out the full list of agencies from the agency file if prompt: - prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") + prompt_successful = TerminalHelper.query_yes_no( + f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}" + ) if prompt_successful or not prompt: logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== MERGED LISTS (transition domain + agency file) ============================" - f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." - f"\n\n{len(merged_agencies)} TOTAL\n\n" + f"\n{TerminalColors.OKGREEN}" + f"\n======================== MERGED LISTS (transition domain + agency file) ============================" + f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." + f"\n\n{len(merged_agencies)} TOTAL\n\n" + ) + self.print_agency_list( + merged_transition_agencies, "Merged_Transition_Domain_Agency_List" ) - self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List") \ No newline at end of file diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 3878a3366..b3902d57e 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -64,9 +64,9 @@ class Command(BaseCommand): ) parser.add_argument( - "--infer_filenames", + "--infer_filenames", action=argparse.BooleanOptionalAction, - help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." + help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.", ) parser.add_argument( @@ -74,7 +74,7 @@ class Command(BaseCommand): ) parser.add_argument( "--domain_contacts_filename", - help="Data file with domain contact information" + help="Data file with domain contact information", ) parser.add_argument( "--contacts_filename", @@ -82,7 +82,7 @@ class Command(BaseCommand): ) parser.add_argument( "--domain_statuses_filename", - help="Data file with domain status information" + help="Data file with domain status information", ) parser.add_argument( "--agency_adhoc_filename", @@ -347,10 +347,12 @@ class Command(BaseCommand): # If it does, update the options options[key] = value except Exception as err: - logger.error(f"""{TerminalColors.FAIL}There was an error loading the JSON responsible + logger.error( + f"""{TerminalColors.FAIL}There was an error loading the JSON responsible for providing filepaths. {TerminalColors.ENDC} - """) + """ + ) raise err sep = args.sep @@ -369,33 +371,36 @@ class Command(BaseCommand): ) # set to 0 to parse all entries ## Variables for Additional TransitionDomain Information ## - # Main script filenames - these do not have defaults domain_contacts_filename = None try: - domain_contacts_filename = directory + options.get("domain_contacts_filename") + domain_contacts_filename = directory + options.get( + "domain_contacts_filename" + ) except TypeError as err: logger.error( - f"Invalid filename of '{args.domain_contacts_filename}'" + f"Invalid filename of '{args.domain_contacts_filename}'" " was provided for domain_contacts_filename" ) - + contacts_filename = None try: contacts_filename = directory + options.get("contacts_filename") except TypeError as err: logger.error( - f"Invalid filename of '{args.contacts_filename}'" + f"Invalid filename of '{args.contacts_filename}'" " was provided for contacts_filename" ) domain_statuses_filename = None try: - domain_statuses_filename = directory + options.get("domain_statuses_filename") + domain_statuses_filename = directory + options.get( + "domain_statuses_filename" + ) except TypeError as err: logger.error( - f"Invalid filename of '{args.domain_statuses_filename}'" + f"Invalid filename of '{args.domain_statuses_filename}'" " was provided for domain_statuses_filename" ) @@ -468,7 +473,10 @@ class Command(BaseCommand): new_entry_email = "" new_entry_emailSent = False # set to False by default - TerminalHelper.print_conditional(True, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") + TerminalHelper.print_conditional( + True, + f"Processing item {total_rows_parsed}: {new_entry_domain_name}", + ) # PART 1: Get the status if new_entry_domain_name not in domain_status_dictionary: @@ -608,7 +616,6 @@ class Command(BaseCommand): ) self.print_summary_status_findings(domains_without_status, outlier_statuses) - logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 96bf0517b..b34a24375 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -28,8 +28,7 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): - help = """ """ # TODO: update this! - + help = """ """ # TODO: update this! # ====================================================== # ================== ARGUMENTS =================== @@ -45,7 +44,7 @@ class Command(BaseCommand): The location of the files used for load_transition_domain migration script EXAMPLE USAGE: > --migrationDirectory /app/tmp - + --migrationJSON The name of the JSON file used for load_transition_domain migration script EXAMPLE USAGE: @@ -129,7 +128,6 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - # ====================================================== # =============== DATA ANALYSIS ================== # ====================================================== @@ -252,7 +250,6 @@ class Command(BaseCommand): """ ) - # ====================================================== # ================= MIGRATIONS =================== # ====================================================== @@ -266,17 +263,13 @@ class Command(BaseCommand): prompts_enabled: bool, debug_max_entries_to_parse: int, ): - if file_directory and file_directory[-1] != "/": file_directory += "/" json_filepath = migration_json_filename """Runs the load_transition_domain script""" # Create the command string command_script = "load_transition_domain" - command_string = ( - f"./manage.py {command_script} " - f"{json_filepath} " - ) + command_string = f"./manage.py {command_script} " f"{json_filepath} " if sep is not None and sep != "|": command_string += f"--sep {sep} " if reset_table: @@ -306,7 +299,7 @@ class Command(BaseCommand): resetTable=reset_table, debug=debug_on, limitParse=debug_max_entries_to_parse, - directory=file_directory + directory=file_directory, ) def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): @@ -326,7 +319,7 @@ class Command(BaseCommand): ) # TODO: make this somehow run inside TerminalHelper prompt if proceed or not prompts_enabled: - call_command(command_script) + call_command(command_script) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index a4e81abf5..413745c61 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -39,7 +39,6 @@ class Command(BaseCommand): help="Sets max number of entries to load, set to 0 to load all entries", ) - # ====================================================== # ===================== PRINTING ====================== # ====================================================== @@ -66,13 +65,14 @@ class Command(BaseCommand): {TerminalColors.ENDC} """, ) - - def parse_limit_reached(self, - debug_max_entries_to_parse: bool, - total_rows_parsed: int - ) -> bool: - if (debug_max_entries_to_parse > 0 - and total_rows_parsed >= debug_max_entries_to_parse): + + def parse_limit_reached( + self, debug_max_entries_to_parse: bool, total_rows_parsed: int + ) -> bool: + if ( + debug_max_entries_to_parse > 0 + and total_rows_parsed >= debug_max_entries_to_parse + ): logger.info( f"""{TerminalColors.YELLOW} ----PARSE LIMIT REACHED. HALTING PARSER.---- @@ -81,7 +81,7 @@ class Command(BaseCommand): ) return True return False - + def print_summary_of_findings( self, domains_to_create, @@ -156,16 +156,15 @@ class Command(BaseCommand): """, ) - # ====================================================== # =================== DOMAIN ===================== # ====================================================== - def update_or_create_domain(self, - transition_domain: TransitionDomain, - debug_on: bool) -> (Domain, bool): - """ Given a transition domain, either finds & updates an existing + def update_or_create_domain( + self, transition_domain: TransitionDomain, debug_on: bool + ) -> (Domain, bool): + """Given a transition domain, either finds & updates an existing corresponding domain, or creates a new corresponding domain in - the Domain table. + the Domain table. Returns the corresponding Domain object and a boolean that is TRUE if that Domain was newly created. @@ -176,7 +175,7 @@ class Command(BaseCommand): transition_domain_status = transition_domain.status transition_domain_creation_date = transition_domain.epp_creation_date transition_domain_expiration_date = transition_domain.epp_expiration_date - + domain_exists = Domain.objects.filter(name=transition_domain_name).exists() if domain_exists: try: @@ -197,7 +196,7 @@ class Command(BaseCommand): transition_domain, target_domain, debug_on ) # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. - + # update dates (creation and expiration) if transition_domain_creation_date is not None: # TODO: added this because I ran into a situation where the created_at date was null (violated a key constraint). How do we want to handle this case? @@ -257,7 +256,6 @@ class Command(BaseCommand): expiration_date=transition_domain_expiration_date, ) return (target_domain, True) - def update_domain_status( self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool @@ -289,7 +287,6 @@ class Command(BaseCommand): return True return False - # ====================================================== # ================ DOMAIN INVITATION ================== # ====================================================== @@ -336,23 +333,27 @@ class Command(BaseCommand): # ====================================================== # ================ DOMAIN INFORMATION ================= # ====================================================== - def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: + def update_domain_information( + self, current: DomainInformation, target: DomainInformation, debug_on: bool + ) -> bool: # DEBUG: TerminalHelper.print_conditional( debug_on, - (f"{TerminalColors.OKCYAN}" - f"Updating: {current}" - f"{TerminalColors.ENDC}"), # noqa + ( + f"{TerminalColors.OKCYAN}" + f"Updating: {current}" + f"{TerminalColors.ENDC}" + ), # noqa ) updated = False fields_to_update = [ - 'organization_type', - 'federal_type', - 'federal_agency', - "organization_name" - ] + "organization_type", + "federal_type", + "federal_agency", + "organization_name", + ] defaults = {field: getattr(target, field) for field in fields_to_update} if current != target: current = target @@ -360,14 +361,19 @@ class Command(BaseCommand): updated = True return updated - + def try_add_domain_information(self): pass - def create_new_domain_info(self, - transition_domain: TransitionDomain, - domain: Domain) -> DomainInformation: - + def create_new_domain_info( + self, + transition_domain: TransitionDomain, + domain: Domain, + agency_choices, + fed_choices, + org_choices, + debug_on, + ) -> DomainInformation: org_type = transition_domain.organization_type fed_type = transition_domain.federal_type fed_agency = transition_domain.federal_agency @@ -387,77 +393,95 @@ class Command(BaseCommand): org_type = ("city", "City") case "Independent Intrastate": org_type = ("special_district", "Special district") - - valid_org_type = org_type in [(name, value) for name, value in DomainApplication.OrganizationChoices.choices] - valid_fed_type = fed_type in [value for name, value in DomainApplication.BranchChoices.choices] - valid_fed_agency = fed_agency in DomainApplication.AGENCIES + + valid_org_type = org_type in org_choices + valid_fed_type = fed_type in fed_choices + valid_fed_agency = fed_agency in agency_choices default_creator, _ = User.objects.get_or_create(username="System") new_domain_info_data = { - 'domain': domain, - 'organization_name': transition_domain.organization_name, + "domain": domain, + "organization_name": transition_domain.organization_name, "creator": default_creator, } - + if valid_org_type: - new_domain_info_data['organization_type'] = org_type[0] - else: + new_domain_info_data["organization_type"] = org_type[0] + elif debug_on: logger.debug(f"No org type found on {domain.name}") if valid_fed_type: - new_domain_info_data['federal_type'] = fed_type.lower() - pass - else: + new_domain_info_data["federal_type"] = fed_type.lower() + elif debug_on: logger.debug(f"No federal type found on {domain.name}") if valid_fed_agency: - new_domain_info_data['federal_agency'] = fed_agency - else: + new_domain_info_data["federal_agency"] = fed_agency + elif debug_on: logger.debug(f"No federal agency found on {domain.name}") new_domain_info = DomainInformation(**new_domain_info_data) - # DEBUG: + # DEBUG: TerminalHelper.print_conditional( True, - (f"{TerminalColors.MAGENTA}" - f"Created Domain Information template for: {new_domain_info}" - f"{TerminalColors.ENDC}"), # noqa + ( + f"{TerminalColors.MAGENTA}" + f"Created Domain Information template for: {new_domain_info}" + f"{TerminalColors.ENDC}" + ), # noqa ) return new_domain_info - def update_or_create_domain_information(self, - transition_domain: TransitionDomain, - debug_on: bool) -> (DomainInformation, bool): - + def update_or_create_domain_information( + self, + transition_domain: TransitionDomain, + agency_choices, + fed_choices, + org_choices, + debug_on: bool, + ) -> (DomainInformation, bool): transition_domain_name = transition_domain.domain_name - + # Get associated domain domain_data = Domain.objects.filter(name=transition_domain.domain_name) if not domain_data.exists(): logger.warn( - f"{TerminalColors.FAIL}" - f"WARNING: No Domain exists for:" - f"{transition_domain_name}" - f"{TerminalColors.ENDC}\n" - ) + f"{TerminalColors.FAIL}" + f"WARNING: No Domain exists for:" + f"{transition_domain_name}" + f"{TerminalColors.ENDC}\n" + ) return (None, None, False) domain = domain_data.get() - template_domain_information = self.create_new_domain_info(transition_domain, domain) + template_domain_information = self.create_new_domain_info( + transition_domain, + domain, + agency_choices, + fed_choices, + org_choices, + debug_on, + ) target_domain_information = None - domain_information_exists = DomainInformation.objects.filter(domain__name=transition_domain_name).exists() + domain_information_exists = DomainInformation.objects.filter( + domain__name=transition_domain_name + ).exists() if domain_information_exists: try: # get the existing domain information object - target_domain_information = DomainInformation.objects.get(domain__name=transition_domain_name) + target_domain_information = DomainInformation.objects.get( + domain__name=transition_domain_name + ) # DEBUG: TerminalHelper.print_conditional( debug_on, - (f"{TerminalColors.FAIL}" - f"Found existing entry in Domain Information table for:" - f"{transition_domain_name}" - f"{TerminalColors.ENDC}"), # noqa + ( + f"{TerminalColors.FAIL}" + f"Found existing entry in Domain Information table for:" + f"{transition_domain_name}" + f"{TerminalColors.ENDC}" + ), # noqa ) # for existing entry, update the status to @@ -466,7 +490,7 @@ class Command(BaseCommand): target_domain_information, template_domain_information, debug_on ) # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. - + return (target_domain_information, domain, False) except DomainInformation.MultipleObjectsReturned: # This should never happen (just like with the Domain Table). @@ -493,15 +517,15 @@ class Command(BaseCommand): # DEBUG: TerminalHelper.print_conditional( debug_on, - (f"{TerminalColors.OKCYAN}" - f"Adding domain information for: " - f"{transition_domain_name}" - f"{TerminalColors.ENDC}"), + ( + f"{TerminalColors.OKCYAN}" + f"Adding domain information for: " + f"{transition_domain_name}" + f"{TerminalColors.ENDC}" + ), ) return (target_domain_information, domain, True) - - # ====================================================== # ===================== HANDLE ======================== # ====================================================== @@ -536,7 +560,6 @@ class Command(BaseCommand): # domain invitations to ADD domain_invitations_to_create = [] - # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 @@ -566,7 +589,7 @@ class Command(BaseCommand): TerminalHelper.print_conditional( debug_on, f"{TerminalColors.OKCYAN}" - "Processing Transition Domain: " + "Processing Transition Domain: " f"{transition_domain_name}, {transition_domain_status}, {transition_domain_email}" f", {transition_domain_creation_date}, {transition_domain_expiration_date}" f"{TerminalColors.ENDC}", # noqa @@ -574,7 +597,9 @@ class Command(BaseCommand): # ====================================================== # ====================== DOMAIN ======================= - target_domain, was_created = self.update_or_create_domain(transition_domain, debug_on) + target_domain, was_created = self.update_or_create_domain( + transition_domain, debug_on + ) debug_string = "" if target_domain is None: @@ -603,12 +628,12 @@ class Command(BaseCommand): # ---------------- UPDATED ---------------- updated_domain_entries.append(transition_domain.domain_name) debug_string = f"updated domain: {target_domain}" - + # DEBUG: TerminalHelper.print_conditional( debug_on, (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), - ) + ) # ====================================================== # ================ DOMAIN INVITATIONS ================== @@ -633,26 +658,36 @@ class Command(BaseCommand): if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): break - # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) - #DomainInvitation.objects.bulk_create(domain_invitations_to_create) + # DomainInvitation.objects.bulk_create(domain_invitations_to_create) # TODO: this is to resolve an error where bulk_create # doesn't save to database in a way that invitation objects can - # utilize. + # utilize. # Then, create DomainInvitation objects for invitation in domain_invitations_to_create: - logger.info(f"Pairing invite to its domain...{invitation}") + if debug_on: + logger.info(f"Pairing invite to its domain...{invitation}") existing_domain = Domain.objects.filter(name=invitation.domain.name) # Make sure the related Domain object is saved if existing_domain.exists(): invitation.domain = existing_domain.get() else: # Raise an err for now - raise Exception(f"Domain {existing_domain} wants to be added but doesn't exist in the DB") + raise Exception( + f"Domain {existing_domain} wants to be added but doesn't exist in the DB" + ) invitation.save() + valid_org_choices = [ + (name, value) + for name, value in DomainApplication.OrganizationChoices.choices + ] + valid_fed_choices = [ + value for name, value in DomainApplication.BranchChoices.choices + ] + valid_agency_choices = DomainApplication.AGENCIES # ====================================================== # ================= DOMAIN INFORMATION ================= logger.info( @@ -661,31 +696,54 @@ class Command(BaseCommand): {TerminalColors.ENDC}""" ) for transition_domain in TransitionDomain.objects.all(): - target_domain_information, associated_domain, was_created = self.update_or_create_domain_information(transition_domain, debug_on) + ( + target_domain_information, + associated_domain, + was_created, + ) = self.update_or_create_domain_information( + transition_domain, + valid_agency_choices, + valid_fed_choices, + valid_org_choices, + debug_on, + ) debug_string = "" if target_domain_information is None: # ---------------- SKIPPED ---------------- skipped_domain_information_entries.append(target_domain_information) - debug_string = f"skipped domain information: {target_domain_information}" + debug_string = ( + f"skipped domain information: {target_domain_information}" + ) elif was_created: - # DEBUG: + # DEBUG: TerminalHelper.print_conditional( debug_on, - (f"{TerminalColors.OKCYAN}" - f"Checking duplicates for: {target_domain_information}" - f"{TerminalColors.ENDC}"), # noqa + ( + f"{TerminalColors.OKCYAN}" + f"Checking duplicates for: {target_domain_information}" + f"{TerminalColors.ENDC}" + ), # noqa ) # ---------------- DUPLICATE ---------------- # The unique key constraint does not allow multiple domain # information objects to share the same domain existing_domain_information_in_to_create = next( - (x for x in domain_information_to_create if x.domain.name == target_domain_information.domain.name), + ( + x + for x in domain_information_to_create + if x.domain.name == target_domain_information.domain.name + ), None, ) # TODO: this is redundant. Currently debugging....running into unique key constraint error.... - existing_domain_info = DomainInformation.objects.filter(domain__name=target_domain_information.domain.name).exists() - if existing_domain_information_in_to_create is not None or existing_domain_info: + existing_domain_info = DomainInformation.objects.filter( + domain__name=target_domain_information.domain.name + ).exists() + if ( + existing_domain_information_in_to_create is not None + or existing_domain_info + ): debug_string = f"""{TerminalColors.YELLOW} Duplicate Detected: {existing_domain_information_in_to_create}. Cannot add duplicate Domain Information object @@ -693,19 +751,23 @@ class Command(BaseCommand): else: # ---------------- CREATED ---------------- domain_information_to_create.append(target_domain_information) - debug_string = f"created domain information: {target_domain_information}" + debug_string = ( + f"created domain information: {target_domain_information}" + ) elif not was_created: # ---------------- UPDATED ---------------- updated_domain_information.append(target_domain_information) - debug_string = f"updated domain information: {target_domain_information}" + debug_string = ( + f"updated domain information: {target_domain_information}" + ) else: debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}" - + # DEBUG: TerminalHelper.print_conditional( debug_on, (f"{TerminalColors.OKCYAN}{debug_string}{TerminalColors.ENDC}"), - ) + ) # ------------------ Parse limit reached? ------------------ # Check parse limit and exit loop if parse limit has been reached @@ -713,11 +775,13 @@ class Command(BaseCommand): break TerminalHelper.print_conditional( - debug_on, - (f"{TerminalColors.YELLOW}" - f"Trying to add: {domain_information_to_create}" - f"{TerminalColors.ENDC}"), - ) + debug_on, + ( + f"{TerminalColors.YELLOW}" + f"Trying to add: {domain_information_to_create}" + f"{TerminalColors.ENDC}" + ), + ) DomainInformation.objects.bulk_create(domain_information_to_create) self.print_summary_of_findings( diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 36b5e3f17..3fe170574 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -15,69 +15,69 @@ from typing import List, Optional class AgencyAdhoc: """Defines the structure given in the AGENCY_ADHOC file""" - agencyid: Optional[int] = field(default=None, repr=True) - agencyname: Optional[str] = field(default=None, repr=True) - active: Optional[str] = field(default=None, repr=True) - isfederal: Optional[str] = field(default=None, repr=True) + agencyid: Optional[int] = field(default=None, repr=True) + agencyname: Optional[str] = field(default=None, repr=True) + active: Optional[str] = field(default=None, repr=True) + isfederal: Optional[str] = field(default=None, repr=True) @dataclass class DomainAdditionalData: """Defines the structure given in the DOMAIN_ADDITIONAL file""" - domainname: Optional[str] = field(default=None, repr=True) - domaintypeid: Optional[int] = field(default=None, repr=True) - authorityid: Optional[int] = field(default=None, repr=True) - orgid: Optional[int] = field(default=None, repr=True) - securitycontactemail: Optional[str] = field(default=None, repr=True) - dnsseckeymonitor: Optional[str] = field(default=None, repr=True) - domainpurpose: Optional[str] = field(default=None, repr=True) + domainname: Optional[str] = field(default=None, repr=True) + domaintypeid: Optional[int] = field(default=None, repr=True) + authorityid: Optional[int] = field(default=None, repr=True) + orgid: Optional[int] = field(default=None, repr=True) + securitycontactemail: Optional[str] = field(default=None, repr=True) + dnsseckeymonitor: Optional[str] = field(default=None, repr=True) + domainpurpose: Optional[str] = field(default=None, repr=True) @dataclass class DomainTypeAdhoc: """Defines the structure given in the DOMAIN_ADHOC file""" - domaintypeid: Optional[int] = field(default=None, repr=True) - domaintype: Optional[str] = field(default=None, repr=True) - code: Optional[str] = field(default=None, repr=True) - active: Optional[str] = field(default=None, repr=True) + domaintypeid: Optional[int] = field(default=None, repr=True) + domaintype: Optional[str] = field(default=None, repr=True) + code: Optional[str] = field(default=None, repr=True) + active: Optional[str] = field(default=None, repr=True) @dataclass class OrganizationAdhoc: """Defines the structure given in the ORGANIZATION_ADHOC file""" - orgid: Optional[int] = field(default=None, repr=True) - orgname: Optional[str] = field(default=None, repr=True) - orgstreet: Optional[str] = field(default=None, repr=True) - orgcity: Optional[str] = field(default=None, repr=True) - orgstate: Optional[str] = field(default=None, repr=True) - orgzip: Optional[str] = field(default=None, repr=True) - orgcountrycode: Optional[str] = field(default=None, repr=True) + orgid: Optional[int] = field(default=None, repr=True) + orgname: Optional[str] = field(default=None, repr=True) + orgstreet: Optional[str] = field(default=None, repr=True) + orgcity: Optional[str] = field(default=None, repr=True) + orgstate: Optional[str] = field(default=None, repr=True) + orgzip: Optional[str] = field(default=None, repr=True) + orgcountrycode: Optional[str] = field(default=None, repr=True) @dataclass class AuthorityAdhoc: """Defines the structure given in the AUTHORITY_ADHOC file""" - authorityid: Optional[int] = field(default=None, repr=True) - firstname: Optional[str] = field(default=None, repr=True) - middlename: Optional[str] = field(default=None, repr=True) - lastname: Optional[str] = field(default=None, repr=True) - email: Optional[str] = field(default=None, repr=True) - phonenumber: Optional[str] = field(default=None, repr=True) - agencyid: Optional[int] = field(default=None, repr=True) - addlinfo: Optional[List[str]] = field(default=None, repr=True) + authorityid: Optional[int] = field(default=None, repr=True) + firstname: Optional[str] = field(default=None, repr=True) + middlename: Optional[str] = field(default=None, repr=True) + lastname: Optional[str] = field(default=None, repr=True) + email: Optional[str] = field(default=None, repr=True) + phonenumber: Optional[str] = field(default=None, repr=True) + agencyid: Optional[int] = field(default=None, repr=True) + addlinfo: Optional[List[str]] = field(default=None, repr=True) @dataclass class DomainEscrow: """Defines the structure given in the DOMAIN_ESCROW file""" - domainname: Optional[str] = field(default=None, repr=True) - creationdate: Optional[date] = field(default=None, repr=True) - expirationdate: Optional[date] = field(default=None, repr=True) + domainname: Optional[str] = field(default=None, repr=True) + creationdate: Optional[date] = field(default=None, repr=True) + expirationdate: Optional[date] = field(default=None, repr=True) class EnumFilenames(Enum): @@ -89,27 +89,12 @@ class EnumFilenames(Enum): # We are sourcing data from many different locations, so its better to track this # as an Enum rather than multiple spread out variables. # We store the "type" as [0], and we store the "default_filepath" as [1]. - AGENCY_ADHOC = ( - "agency_adhoc", - "agency.adhoc.dotgov.txt" - ) + AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt") DOMAIN_ADDITIONAL = ( "domain_additional", "domainadditionaldatalink.adhoc.dotgov.txt", ) - DOMAIN_ESCROW = ( - "domain_escrow", - "escrow_domains.daily.dotgov.GOV.txt" - ) - DOMAIN_ADHOC = ( - "domain_adhoc", - "domaintypes.adhoc.dotgov.txt" - ) - ORGANIZATION_ADHOC = ( - "organization_adhoc", - "organization.adhoc.dotgov.txt" - ) - AUTHORITY_ADHOC = ( - "authority_adhoc", - "authority.adhoc.dotgov.txt" - ) + DOMAIN_ESCROW = ("domain_escrow", "escrow_domains.daily.dotgov.GOV.txt") + DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt") + ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt") + AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt") diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index fd7752839..b06bc5299 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -69,10 +69,15 @@ class FileTransitionLog: log = self.LogItem(file_type, code, message, domain_name) dict_name = (file_type, domain_name) self._add_to_log_list(dict_name, log) - def create_log_item( - self, file_type, code, message, domain_name=None, add_to_list=True, minimal_logging=True + self, + file_type, + code, + message, + domain_name=None, + add_to_list=True, + minimal_logging=True, ): """Creates and returns an LogItem object. @@ -81,10 +86,10 @@ class FileTransitionLog: log = self.LogItem(file_type, code, message, domain_name) if not add_to_list: return log - + dict_name = (file_type, domain_name) self._add_to_log_list(dict_name, log) - + restrict_type = [] if minimal_logging: restrict_type = [LogCode.INFO, LogCode.WARNING] @@ -99,7 +104,7 @@ class FileTransitionLog: def _add_to_log_list(self, log_name, log): if log_name not in self.logs: self.logs[log_name] = [log] - else: + else: self.logs[log_name].append(log) def display_all_logs(self): @@ -107,9 +112,7 @@ class FileTransitionLog: for parent_log in self.logs: for child_log in parent_log: TerminalHelper.print_conditional( - True, - child_log.message, - child_log.severity + True, child_log.message, child_log.severity ) def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT): @@ -125,16 +128,14 @@ class FileTransitionLog: domain_logs = self.get_logs(file_type, domain_name) if domain_logs is None: return None - + for log in domain_logs: TerminalHelper.print_conditional( - restrict_type != log.code, - log.message, - log.code + restrict_type != log.code, log.message, log.code ) def get_logs(self, file_type, domain_name): - """Grabs the logs associated with + """Grabs the logs associated with a particular file_type and domain_name""" log_name = (file_type, domain_name) return self.logs.get(log_name) @@ -213,19 +214,20 @@ class LoadExtraTransitionDomain: logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - Updated {len(updated_transition_domains)} transition domain entries: - {[domain for domain in updated_transition_domains]} + Updated {len(updated_transition_domains)} transition domain entries {TerminalColors.ENDC} """ ) else: # TODO - update - TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") + TerminalHelper.print_conditional( + self.debug, + f"{TerminalHelper.array_as_string(updated_transition_domains)}", + ) logger.error( f"""{TerminalColors.FAIL} ============= FINISHED WITH ERRORS =============== - Updated {len(updated_transition_domains)} transition domain entries: - {[domain for domain in updated_transition_domains]} + Updated {len(updated_transition_domains)} transition domain entries, Failed to update {failed_count} transition domain entries: {[domain for domain in failed_transition_domains]} {TerminalColors.ENDC} @@ -237,7 +239,8 @@ class LoadExtraTransitionDomain: total_transition_domains = len(updated_transition_domains) total_updates_made = TransitionDomain.objects.all().count() if total_transition_domains != total_updates_made: - logger.error(f"""{TerminalColors.FAIL} + logger.error( + f"""{TerminalColors.FAIL} WARNING: something went wrong processing domain information data. Total Transition Domains expecting a data update: {total_transition_domains} @@ -248,7 +251,8 @@ class LoadExtraTransitionDomain: corrupt data. Please check logs to diagnose. ----- TERMINATING ---- - """) + """ + ) sys.exit() def parse_creation_expiration_data(self, domain_name, transition_domain): @@ -262,19 +266,15 @@ class LoadExtraTransitionDomain: self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ESCROW, LogCode.ERROR, - "Could not add epp_creation_date and epp_expiration_date " + "Could not add epp_creation_date and epp_expiration_date " f"on {domain_name}, no data exists.", domain_name, - not self.debug + not self.debug, ) return transition_domain - creation_exists = ( - transition_domain.epp_creation_date is not None - ) - expiration_exists = ( - transition_domain.epp_expiration_date is not None - ) + creation_exists = transition_domain.epp_creation_date is not None + expiration_exists = transition_domain.epp_expiration_date is not None transition_domain.epp_creation_date = info.creationdate transition_domain.epp_expiration_date = info.expirationdate @@ -311,7 +311,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add federal_agency on {domain_name}, no data exists.", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -326,7 +326,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add inactive agency {info.agencyname} on {domain_name}", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -336,7 +336,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add non-federal agency {info.agencyname} on {domain_name}", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -369,7 +369,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add domain_type on {domain_name}, no data exists.", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -392,7 +392,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -453,7 +453,7 @@ class LoadExtraTransitionDomain: LogCode.ERROR, f"Could not add organization_name on {domain_name}, no data exists.", domain_name, - not self.debug + not self.debug, ) return transition_domain @@ -487,7 +487,7 @@ class LoadExtraTransitionDomain: LogCode.INFO, f"Added {var_name} as '{changed_value}' on {domain_name}", domain_name, - not self.debug + not self.debug, ) else: self.parse_logs.create_log_item( @@ -495,7 +495,7 @@ class LoadExtraTransitionDomain: LogCode.WARNING, f"Updated existing {var_name} to '{changed_value}' on {domain_name}", domain_name, - not self.debug + not self.debug, ) # Property getters, i.e. orgid or domaintypeid @@ -523,7 +523,7 @@ class LoadExtraTransitionDomain: domain_info = self.get_domain_data(domain_name) if domain_info is None: return None - + # The agency record is within the authority adhoc authority_id = domain_info.authorityid authority = self.get_authority_adhoc(authority_id) @@ -542,14 +542,14 @@ class LoadExtraTransitionDomain: return None type_id = domain_info.authorityid return self.get_authority_adhoc(type_id) - + def get_domain_escrow_info(self, domain_name): domain_info = self.get_domain_data(domain_name) if domain_info is None: return None type_id = domain_info.domainname return self.get_domain_escrow(type_id) - + # Object getters, i.e. DomainAdditionalData or OrganizationAdhoc def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, @@ -575,7 +575,7 @@ class LoadExtraTransitionDomain: """Grabs a corresponding row within the AUTHORITY_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.AUTHORITY_ADHOC, desired_id) - + def get_domain_escrow(self, desired_id) -> DomainEscrow: """Grabs a corresponding row within the DOMAIN_ESCROW file, based off a desired_id""" @@ -615,7 +615,9 @@ class LoadExtraTransitionDomain: desired_type = self.parsed_data_container.file_data.get(file_type) if desired_type is None: self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Type {file_type} does not exist", + file_type, + LogCode.ERROR, + f"Type {file_type} does not exist", ) return None @@ -624,10 +626,13 @@ class LoadExtraTransitionDomain: obj = desired_type.data.get(desired_id) if obj is None: self.parse_logs.create_log_item( - file_type, LogCode.ERROR, f"Id {desired_id} does not exist for {file_type.value[0]}" + file_type, + LogCode.ERROR, + f"Id {desired_id} does not exist for {file_type.value[0]}", ) return obj + # TODO - change name @dataclass class FileDataHolder: @@ -698,18 +703,18 @@ class FileDataHolder: # matches, then we shouldn't infer if total_groups == 0 or total_groups > 2: return (self.filename, False) - + # If only one match is returned, # it means that our default matches our request if total_groups == 1: return (self.filename, True) - + # Otherwise, if two are returned, then # its likely the pattern we want date = match.group(1) filename_without_date = match.group(2) - # After stripping out the date, + # After stripping out the date, # do the two filenames match? can_infer = filename_without_date == default_file_name if not can_infer: @@ -861,7 +866,7 @@ class ExtraTransitionDomain: if not infer_filenames: logger.error(f"Could not find file: {filename}") continue - + # Infer filename logic # # This mode is used for internal development use and testing only. Rather than having # to manually define the filename each time, we can infer what the filename @@ -898,26 +903,15 @@ class ExtraTransitionDomain: file_type.data = {} def parse_csv_file( - self, - file, - seperator, - dataclass_type, - id_field, - is_domain_escrow=False + self, file, seperator, dataclass_type, id_field, is_domain_escrow=False ): # Domain escrow is an edge case if is_domain_escrow: - item_to_return = self._read_domain_escrow( - file, - seperator - ) + item_to_return = self._read_domain_escrow(file, seperator) return item_to_return else: item_to_return = self._read_csv_file( - file, - seperator, - dataclass_type, - id_field + file, seperator, dataclass_type, id_field ) return item_to_return @@ -946,14 +940,16 @@ class ExtraTransitionDomain: reader = csv.DictReader(requested_file, delimiter=seperator) for row in reader: # Checks if we encounter any bad data. - # If we do, we (non-destructively) clean the file + # If we do, we (non-destructively) clean the file if None in row: logger.warning( f"{TerminalColors.YELLOW}" f"Found bad data in {file}. Attempting to clean." f"{TerminalColors.ENDC}" ) - updated_file_content = self.replace_bad_seperators(file, f"{seperator}", ";badseperator;") + updated_file_content = self.replace_bad_seperators( + file, f"{seperator}", ";badseperator;" + ) dict_data = {} break @@ -964,7 +960,7 @@ class ExtraTransitionDomain: if id_field == "domainname" and row_id is not None: row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) - + # After we clean the data, try to parse it again if updated_file_content: logger.info( @@ -999,10 +995,10 @@ class ExtraTransitionDomain: row_id = row_id.lower() dict_data[row_id] = dataclass_type(**row) return dict_data - + def replace_bad_seperators(self, filename, delimiter, special_character): with open(filename, "r", encoding="utf-8-sig") as file: contents = file.read() new_content = re.sub(rf" \{delimiter} ", special_character, contents) - return new_content \ No newline at end of file + return new_content diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 0f309f9c8..05c37b271 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -4,9 +4,10 @@ import sys logger = logging.getLogger(__name__) + class LogCode(Enum): """Stores the desired log severity - + Overview of error codes: - 1 ERROR - 2 WARNING @@ -21,6 +22,7 @@ class LogCode(Enum): DEBUG = 4 DEFAULT = 5 + class TerminalColors: """Colors for terminal outputs (makes reading the logs WAY easier)""" @@ -81,7 +83,14 @@ class TerminalHelper: The "answer" return value is True for "yes" or False for "no". """ - valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit"} + valid = { + "yes": True, + "y": True, + "ye": True, + "no": False, + "n": False, + "e": "exit", + } if default is None: prompt = " [y/n] " elif default == "yes": @@ -105,22 +114,20 @@ class TerminalHelper: # @staticmethod def array_as_string(array_to_convert: []) -> str: - array_as_string = "{}".format( - "\n".join(map(str, array_to_convert)) - ) + array_as_string = "{}".format("\n".join(map(str, array_to_convert))) return array_as_string @staticmethod def print_conditional( - print_condition: bool, - print_statement: str, - log_severity: LogCode = LogCode.DEFAULT + print_condition: bool, + print_statement: str, + log_severity: LogCode = LogCode.DEFAULT, ): """This function reduces complexity of debug statements in other functions. It uses the logger to write the given print_statement to the terminal if print_condition is TRUE. - + print_condition: bool -> Prints if print_condition is TRUE print_statement: str -> The statement to print @@ -181,29 +188,40 @@ class TerminalHelper: @staticmethod def get_file_line_count(filepath: str) -> int: - with open(filepath,'r') as file: + with open(filepath, "r") as file: li = file.readlines() total_line = len(li) return total_line @staticmethod - def print_to_file_conditional(print_condition: bool, filename: str, file_directory: str, file_contents: str): - """Sometimes logger outputs get insanely huge. - """ - if (print_condition): + def print_to_file_conditional( + print_condition: bool, filename: str, file_directory: str, file_contents: str + ): + """Sometimes logger outputs get insanely huge.""" + if print_condition: # Add a slash if the last character isn't one if file_directory and file_directory[-1] != "/": file_directory += "/" # Assemble filepath filepath = f"{file_directory}{filename}.txt" # Write to file - logger.info(f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}") + logger.info( + f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}" + ) with open(f"{filepath}", "w+") as f: f.write(file_contents) - @staticmethod - def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"): + def printProgressBar( + iteration, + total, + prefix="Progress:", + suffix="Complete", + decimals=1, + length=100, + fill="█", + printEnd="\r", + ): """ Call in a loop to create terminal progress bar @params: @@ -227,10 +245,12 @@ class TerminalHelper: printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) """ - percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) + percent = ("{0:." + str(decimals) + "f}").format( + 100 * (iteration / float(total)) + ) filledLength = int(length * iteration // total) - bar = fill * filledLength + '-' * (length - filledLength) - print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) + bar = fill * filledLength + "-" * (length - filledLength) + print(f"\r{prefix} |{bar}| {percent}% {suffix}", end=printEnd) # Print New Line on Complete - if iteration == total: - print() \ No newline at end of file + if iteration == total: + print() diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 335f74cd9..11f3afe7c 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -3,19 +3,20 @@ from typing import Optional from registrar.management.commands.utility.epp_data_containers import EnumFilenames + @dataclass class TransitionDomainArguments: """Stores arguments for load_transition_domain, structurally a mix - of a dataclass and a regular class, meaning we get a hardcoded + of a dataclass and a regular class, meaning we get a hardcoded representation of the values we want, while maintaining flexiblity and reducing boilerplate. - + All pre-defined fields are optional but will remain on the model definition. In this event, they are provided a default value if none is given. """ # Maintains an internal kwargs list and sets values - # that match the class definition. + # that match the class definition. def __init__(self, **kwargs): self.kwargs = kwargs for k, v in kwargs.items(): @@ -36,14 +37,26 @@ class TransitionDomainArguments: # Filenames # ## Adhocs ## - agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True) - domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True) - organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True) - authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) + agency_adhoc_filename: Optional[str] = field( + default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True + ) + domain_adhoc_filename: Optional[str] = field( + default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True + ) + organization_adhoc_filename: Optional[str] = field( + default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True + ) + authority_adhoc_filename: Optional[str] = field( + default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True + ) ## Data files ## - domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True) - domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) + domain_escrow_filename: Optional[str] = field( + default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True + ) + domain_additional_filename: Optional[str] = field( + default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True + ) domain_contacts_filename: Optional[str] = field(default=None, repr=True) domain_statuses_filename: Optional[str] = field(default=None, repr=True) contacts_filename: Optional[str] = field(default=None, repr=True) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 877737f43..5c5b0f1f7 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -13,6 +13,7 @@ from registrar.models import ( from django.core.management import call_command from unittest.mock import patch + class TestMigrations(TestCase): def setUp(self): """ """ @@ -46,24 +47,30 @@ class TestMigrations(TestCase): UserDomainRole.objects.all().delete() def run_load_domains(self): - with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", + return_value=True, + ): call_command( "load_transition_domain", self.migration_json_filename, - directory=self.test_data_file_location + directory=self.test_data_file_location, ) def run_transfer_domains(self): call_command("transfer_transition_domains_to_domains") def run_master_script(self): - with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", + return_value=True, + ): call_command( "master_domain_migrations", runMigrations=True, migrationDirectory=self.test_data_file_location, migrationJSON=self.migration_json_filename, - disablePrompts=True + disablePrompts=True, ) def compare_tables( @@ -204,7 +211,7 @@ class TestMigrations(TestCase): expected_missing_domain_informations, expected_missing_domain_invitations, ) - + def test_load_full_transition_domain(self): # Load command self.run_load_domains() @@ -242,7 +249,7 @@ class TestMigrations(TestCase): federal_type="Executive", federal_agency="InnoZ", epp_creation_date=None, - epp_expiration_date=None + epp_expiration_date=None, ), TransitionDomain( username="reginald.ratcliff4@test.com", @@ -254,20 +261,21 @@ class TestMigrations(TestCase): federal_type=None, federal_agency=None, epp_creation_date=None, - epp_expiration_date=None - ) + epp_expiration_date=None, + ), ] - expected_transition_domains = TransitionDomain.objects.filter(username="alexandra.bobbitt5@test.com") + expected_transition_domains = TransitionDomain.objects.filter( + username="alexandra.bobbitt5@test.com" + ) self.assertEqual(expected_transition_domains.count(), 1) expected_transition_domain = expected_transition_domains.get() - #TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") + # TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") # Afterwards, their values should be what we expect all_transition_domains = TransitionDomain.objects.all() for domain in all_transition_domains: for expected in expected_transition_domains: - # This data gets created when the object is, # so we should just match it. Not relevant # to the added data. @@ -277,7 +285,7 @@ class TestMigrations(TestCase): # Each TransitionDomain should have the correct data self.assertEqual(domain, expected) - + def test_load_full_domain(self): self.run_load_domains() self.run_transfer_domains() @@ -323,10 +331,10 @@ class TestMigrations(TestCase): testdomain = testdomain_domains.get() self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) - #self.assertEqual(testdomain.created_at, "test") + # self.assertEqual(testdomain.created_at, "test") self.assertEqual(testdomain.name, "fakewebsite2.gov") self.assertEqual(testdomain.state, "on hold") - + def test_load_full_domain_information(self): self.run_load_domains() self.run_transfer_domains() @@ -355,7 +363,7 @@ class TestMigrations(TestCase): # Test created Domain Information objects domain = Domain.objects.filter(name="anomaly.gov").get() anomaly_domain_infos = DomainInformation.objects.filter(domain=domain) - + self.assertEqual(anomaly_domain_infos.count(), 1) # This domain should be pretty barebones. Something isnt @@ -365,7 +373,7 @@ class TestMigrations(TestCase): self.assertEqual(anomaly.organization_type, None) self.assertEqual(anomaly.federal_agency, None) self.assertEqual(anomaly.federal_type, None) - + # Check for the "system" creator user Users = User.objects.filter(username="System") self.assertEqual(Users.count(), 1) @@ -380,13 +388,12 @@ class TestMigrations(TestCase): self.assertEqual(fakewebsite.organization_type, "federal") self.assertEqual(fakewebsite.federal_agency, "Department of Commerce") self.assertEqual(fakewebsite.federal_type, "executive") - + # Check for the "system" creator user Users = User.objects.filter(username="System") self.assertEqual(Users.count(), 1) self.assertEqual(anomaly.creator, Users.get()) - def test_transfer_transition_domains_to_domains(self): self.run_load_domains() self.run_transfer_domains() diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 1e43a3e24..0db1ff402 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -100,7 +100,7 @@ class DomainPermission(PermissionsLoginMixin): if DomainInformation.objects.filter(id=pk).exists(): requested_domain = DomainInformation.objects.get(id=pk) - # If no domain_application object exists and we are + # If no domain_application object exists and we are # coming from the manage_domain dashboard, this is likely # a transition domain. domain_application = requested_domain.domain_application From 3e3cfde00a1e1342156c002d931f5bcbf5c2b72f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 07:44:29 -0700 Subject: [PATCH 72/88] Some linting and test cases --- .../commands/agency_data_extractor.py | 273 ------------------ .../commands/load_transition_domain.py | 23 +- .../commands/master_domain_migrations.py | 4 +- .../transfer_transition_domains_to_domains.py | 14 +- .../test_transition_domain_migrations.py | 81 +----- 5 files changed, 23 insertions(+), 372 deletions(-) delete mode 100644 src/registrar/management/commands/agency_data_extractor.py diff --git a/src/registrar/management/commands/agency_data_extractor.py b/src/registrar/management/commands/agency_data_extractor.py deleted file mode 100644 index 1f0d6e0da..000000000 --- a/src/registrar/management/commands/agency_data_extractor.py +++ /dev/null @@ -1,273 +0,0 @@ -import argparse -import csv -import logging - -from django.core.management import BaseCommand - -from registrar.management.commands.utility.terminal_helper import ( - TerminalColors, - TerminalHelper, -) -from registrar.models.domain_application import DomainApplication -from registrar.models.transition_domain import TransitionDomain - -logger = logging.getLogger(__name__) - -# DEV SHORTCUT: -# Example command for running this script: -# docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug - - -class Command(BaseCommand): - help = """Loads data for domains that are in transition - (populates transition_domain model objects).""" - - def add_arguments(self, parser): - """Add file that contains agency data""" - parser.add_argument( - "agency_data_filename", help="Data file with agency information" - ) - parser.add_argument("--dir", default="migrationdata", help="Desired directory") - parser.add_argument("--sep", default="|", help="Delimiter character") - - parser.add_argument( - "--debug", - help="Prints additional debug statements to the terminal", - action=argparse.BooleanOptionalAction, - ) - parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) - - @staticmethod - def extract_agencies(agency_data_filepath: str, sep: str, debug: bool) -> [str]: - """Extracts all the agency names from the provided - agency file (skips any duplicates) and returns those - names in an array""" - agency_names = [] - logger.info( - f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}" - ) - with open(agency_data_filepath, "r") as agency_data_filepath: # noqa - for row in csv.reader(agency_data_filepath, delimiter=sep): - agency_name = row[1] - TerminalHelper.print_conditional(debug, f"Checking: {agency_name}") - if agency_name not in agency_names: - agency_names.append(agency_name) - logger.info( - f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}" - ) - return agency_names - - @staticmethod - def compare_agency_lists( - provided_agencies: [str], existing_agencies: [str], debug: bool - ): - """ - Compares new_agencies with existing_agencies and - provides the equivalent of an outer-join on the two - (printed to the terminal) - """ - - new_agencies = [] - # 1 - Get all new agencies that we don't already have (We might want to ADD these to our list) - for agency in provided_agencies: - if agency not in existing_agencies and agency not in new_agencies: - new_agencies.append(agency) - TerminalHelper.print_conditional( - debug, - f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}", - ) - - possibly_unused_agencies = [] - # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) - for agency in existing_agencies: - if ( - agency not in provided_agencies - and agency not in possibly_unused_agencies - ): - possibly_unused_agencies.append(agency) - TerminalHelper.print_conditional( - debug, - f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}", - ) - - matched_agencies = [] - for agency in provided_agencies: - if agency in existing_agencies: - matched_agencies.append(agency) - TerminalHelper.print_conditional( - debug, - f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}", - ) - - # Print the summary of findings - # 1 - Print the list of agencies in the NEW list, which we do not already have - # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? - new_agencies_as_string = "{}".format(",\n ".join(map(str, new_agencies))) - possibly_unused_agencies_as_string = "{}".format( - ",\n ".join(map(str, possibly_unused_agencies)) - ) - matched_agencies_as_string = "{}".format( - ",\n ".join(map(str, matched_agencies)) - ) - - logger.info( - f""" - {TerminalColors.OKGREEN} - ======================== SUMMARY OF FINDINGS ============================ - {len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file. - {len(existing_agencies)} AGENCIES FOUND IN THE TARGETED SYSTEM. - - {len(provided_agencies)-len(new_agencies)} AGENCIES MATCHED - (These are agencies that are in the given agency file AND in our system already) - {TerminalColors.YELLOW}{matched_agencies_as_string} - {TerminalColors.OKGREEN} - - {len(new_agencies)} AGENCIES TO ADD: - These agencies were in the provided agency file, but are not in our system. - {TerminalColors.YELLOW}{new_agencies_as_string} - {TerminalColors.OKGREEN} - - {len(possibly_unused_agencies)} AGENCIES TO (POSSIBLY) REMOVE: - These agencies are in our system, but not in the provided agency file: - {TerminalColors.YELLOW}{possibly_unused_agencies_as_string} - {TerminalColors.ENDC} - """ - ) - - @staticmethod - def print_agency_list(agencies, filename): - full_agency_list_as_string = "{}".format(",\n".join(map(str, agencies))) - logger.info( - f"\n{TerminalColors.YELLOW}" - f"\n{full_agency_list_as_string}" - f"{TerminalColors.OKGREEN}" - ) - logger.info(f"{TerminalColors.MAGENTA}Writing to file...{TerminalColors.ENDC}") - with open(f"tmp/[{filename}].txt", "w+") as f: - f.write(full_agency_list_as_string) - - def handle( - self, - agency_data_filename, - **options, - ): - """Parse the agency data file.""" - - # Get all the arguments - sep = options.get("sep") - debug = options.get("debug") - prompt = options.get("prompt") - dir = options.get("dir") - - agency_data_file = dir + "/" + agency_data_filename - - new_agencies = self.extract_agencies(agency_data_file, sep, debug) - hard_coded_agencies = DomainApplication.AGENCIES - transition_domain_agencies = ( - TransitionDomain.objects.all() - .values_list("federal_agency", flat=True) - .distinct() - ) - print(transition_domain_agencies) - - merged_agencies = new_agencies - for agency in hard_coded_agencies: - if agency not in merged_agencies: - merged_agencies.append(agency) - - merged_transition_agencies = new_agencies - for agency in transition_domain_agencies: - if agency not in merged_transition_agencies: - merged_transition_agencies.append(agency) - - prompt_successful = False - - # OPTION to compare the agency file to our hard-coded list - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - self.compare_agency_lists(new_agencies, hard_coded_agencies, debug) - - # OPTION to compare the agency file to Transition Domains - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - self.compare_agency_lists(new_agencies, transition_domain_agencies, debug) - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF IMPORTED AGENCIES ============================" - f"\nThese are all the agencies provided by the given agency file." - f"\n\n{len(new_agencies)} TOTAL\n\n" - ) - self.print_agency_list(new_agencies, "Imported_Agencies") - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF AGENCIES IN DROPDOWN ============================" - f"\nThese are all the agencies hard-coded in our system for the dropdown list." - f"\n\n{len(hard_coded_agencies)} TOTAL\n\n" - ) - self.print_agency_list(hard_coded_agencies, "Dropdown_Agencies") - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== FULL LIST OF AGENCIES IN TRANSITION DOMAIN ============================" - f"\nThese are all the agencies in the Transition Domains table." - f"\n\n{len(transition_domain_agencies)} TOTAL\n\n" - ) - self.print_agency_list( - transition_domain_agencies, "Transition_Domain_Agencies" - ) - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== MERGED LISTS (dropdown + agency file) ============================" - f"\nThese are all the agencies our dropdown plus all the agencies in the agency file." - f"\n\n{len(merged_agencies)} TOTAL\n\n" - ) - self.print_agency_list(merged_agencies, "Merged_Dropdown_Agency_List") - - # OPTION to print out the full list of agencies from the agency file - if prompt: - prompt_successful = TerminalHelper.query_yes_no( - f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}" - ) - if prompt_successful or not prompt: - logger.info( - f"\n{TerminalColors.OKGREEN}" - f"\n======================== MERGED LISTS (transition domain + agency file) ============================" - f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." - f"\n\n{len(merged_agencies)} TOTAL\n\n" - ) - self.print_agency_list( - merged_transition_agencies, "Merged_Transition_Domain_Agency_List" - ) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index b3902d57e..bb6f59ad9 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -66,7 +66,8 @@ class Command(BaseCommand): parser.add_argument( "--infer_filenames", action=argparse.BooleanOptionalAction, - help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.", + help="Determines if we should infer filenames or not." + "Recommended to be enabled only in a development or testing setting.", ) parser.add_argument( @@ -331,7 +332,7 @@ class Command(BaseCommand): directory += "/" json_filepath = directory + migration_json_filename - ### Process JSON file ### + # Process JSON file # # If a JSON was provided, use its values instead of defaults. # TODO: there is no way to discern user overrides from those arg’s defaults. with open(json_filepath, "r") as jsonFile: @@ -339,7 +340,6 @@ class Command(BaseCommand): try: data = json.load(jsonFile) # Create an instance of TransitionDomainArguments - has_desired_args = False # Iterate over the data from the JSON file for key, value in data.items(): # Check if the key exists in TransitionDomainArguments @@ -348,10 +348,10 @@ class Command(BaseCommand): options[key] = value except Exception as err: logger.error( - f"""{TerminalColors.FAIL}There was an error loading the JSON responsible - for providing filepaths. - {TerminalColors.ENDC} - """ + f"{TerminalColors.FAIL}" + "There was an error loading " + "the JSON responsible for providing filepaths." + f"{TerminalColors.ENDC}" ) raise err @@ -370,7 +370,7 @@ class Command(BaseCommand): args.limitParse ) # set to 0 to parse all entries - ## Variables for Additional TransitionDomain Information ## + # Variables for Additional TransitionDomain Information # # Main script filenames - these do not have defaults domain_contacts_filename = None @@ -378,7 +378,7 @@ class Command(BaseCommand): domain_contacts_filename = directory + options.get( "domain_contacts_filename" ) - except TypeError as err: + except TypeError: logger.error( f"Invalid filename of '{args.domain_contacts_filename}'" " was provided for domain_contacts_filename" @@ -387,7 +387,7 @@ class Command(BaseCommand): contacts_filename = None try: contacts_filename = directory + options.get("contacts_filename") - except TypeError as err: + except TypeError: logger.error( f"Invalid filename of '{args.contacts_filename}'" " was provided for contacts_filename" @@ -398,7 +398,7 @@ class Command(BaseCommand): domain_statuses_filename = directory + options.get( "domain_statuses_filename" ) - except TypeError as err: + except TypeError: logger.error( f"Invalid filename of '{args.domain_statuses_filename}'" " was provided for domain_statuses_filename" @@ -458,7 +458,6 @@ class Command(BaseCommand): # Start parsing the main file and create TransitionDomain objects logger.info("Reading domain-contacts data file %s", domain_contacts_filename) - total_lines = TerminalHelper.get_file_line_count(domain_contacts_filename) with open(domain_contacts_filename, "r") as domain_contacts_file: for row in csv.reader(domain_contacts_file, delimiter=sep): # TerminalHelper.printProgressBar(total_rows_parsed, total_lines) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index b34a24375..95349ef45 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -7,7 +7,6 @@ import logging import argparse -import sys from django.core.management import BaseCommand from django.core.management import call_command @@ -89,7 +88,8 @@ class Command(BaseCommand): # The following file arguments have default values for running in the sandbox - # TODO: make this a mandatory argument (if/when we strip out defaults, it will be mandatory) + # TODO: make this a mandatory argument + # (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( "--migrationJSON", diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 413745c61..af7976c30 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -114,7 +114,7 @@ class Command(BaseCommand): Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) {TerminalColors.ENDC} - """ + """ # noqa ) if len(skipped_domain_entries) > 0: logger.info( @@ -192,15 +192,19 @@ class Command(BaseCommand): # ---- UPDATE THE DOMAIN # update the status - update_made = self.update_domain_status( + self.update_domain_status( transition_domain, target_domain, debug_on ) - # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. + # TODO: not all domains need to be updated (the information is the same). + # Need to bubble this up to the final report. # update dates (creation and expiration) if transition_domain_creation_date is not None: - # TODO: added this because I ran into a situation where the created_at date was null (violated a key constraint). How do we want to handle this case? + # TODO: added this because I ran into a situation where + # the created_at date was null (violated a key constraint). + # How do we want to handle this case? target_domain.created_at = transition_domain_creation_date + if transition_domain_expiration_date is not None: target_domain.expiration_date = transition_domain_expiration_date target_domain.save() @@ -486,7 +490,7 @@ class Command(BaseCommand): # for existing entry, update the status to # the transition domain status - update_made = self.update_domain_information( + self.update_domain_information( target_domain_information, template_domain_information, debug_on ) # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 5c5b0f1f7..fb93e52b6 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -212,80 +212,6 @@ class TestMigrations(TestCase): expected_missing_domain_invitations, ) - def test_load_full_transition_domain(self): - # Load command - self.run_load_domains() - - # We should get a consistent number - # of records - expected_total_transition_domains = 9 - expected_total_domains = 0 - expected_total_domain_informations = 0 - expected_total_domain_invitations = 0 - - expected_missing_domains = 9 - expected_duplicate_domains = 0 - expected_missing_domain_informations = 9 - expected_missing_domain_invitations = 9 - self.compare_tables( - expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) - - expected_transition_domains = [ - TransitionDomain( - username="alexandra.bobbitt5@test.com", - domain_name="fakewebsite2.gov", - status="on hold", - email_sent=False, - organization_type="Federal", - organization_name="Fanoodle", - federal_type="Executive", - federal_agency="InnoZ", - epp_creation_date=None, - epp_expiration_date=None, - ), - TransitionDomain( - username="reginald.ratcliff4@test.com", - domain_name="fakewebsite3.gov", - status="ready", - email_sent=False, - organization_type="City", - organization_name="Sushi", - federal_type=None, - federal_agency=None, - epp_creation_date=None, - epp_expiration_date=None, - ), - ] - - expected_transition_domains = TransitionDomain.objects.filter( - username="alexandra.bobbitt5@test.com" - ) - self.assertEqual(expected_transition_domains.count(), 1) - expected_transition_domain = expected_transition_domains.get() - - # TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") - # Afterwards, their values should be what we expect - all_transition_domains = TransitionDomain.objects.all() - for domain in all_transition_domains: - for expected in expected_transition_domains: - # This data gets created when the object is, - # so we should just match it. Not relevant - # to the added data. - expected.id = domain.id - expected.created_at = domain.created_at - expected.updated_at = domain.updated_at - - # Each TransitionDomain should have the correct data - self.assertEqual(domain, expected) - def test_load_full_domain(self): self.run_load_domains() self.run_transfer_domains() @@ -317,11 +243,7 @@ class TestMigrations(TestCase): anomaly = anomaly_domains.get() self.assertEqual(anomaly.expiration_date, datetime.date(2023, 3, 9)) - """ - self.assertEqual( - anomaly.created_at, datetime.datetime(2023, 11, 8, 17, 23, 46, 764663, tzinfo=datetime.timezone.utc) - ) - """ + self.assertEqual(anomaly.name, "anomaly.gov") self.assertEqual(anomaly.state, "ready") @@ -331,7 +253,6 @@ class TestMigrations(TestCase): testdomain = testdomain_domains.get() self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) - # self.assertEqual(testdomain.created_at, "test") self.assertEqual(testdomain.name, "fakewebsite2.gov") self.assertEqual(testdomain.state, "on hold") From 5b4a357069030530c1ef15f623861ff5bf9349f2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:47:59 -0700 Subject: [PATCH 73/88] Linting --- .../commands/load_transition_domain.py | 2 +- .../commands/master_domain_migrations.py | 2 +- .../transfer_transition_domains_to_domains.py | 34 ++++++++++++------- .../commands/utility/epp_data_containers.py | 2 +- .../utility/extra_transition_domain_helper.py | 21 +++++++----- .../commands/utility/terminal_helper.py | 10 +++--- .../utility/transition_domain_arguments.py | 4 +-- .../test_transition_domain_migrations.py | 8 +++-- 8 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index bb6f59ad9..20c0c34e1 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -66,7 +66,7 @@ class Command(BaseCommand): parser.add_argument( "--infer_filenames", action=argparse.BooleanOptionalAction, - help="Determines if we should infer filenames or not." + help="Determines if we should infer filenames or not." "Recommended to be enabled only in a development or testing setting.", ) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 95349ef45..6656458d7 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -88,7 +88,7 @@ class Command(BaseCommand): # The following file arguments have default values for running in the sandbox - # TODO: make this a mandatory argument + # TODO: make this a mandatory argument # (if/when we strip out defaults, it will be mandatory) # TODO: use the migration directory arg or force user to type FULL filepath? parser.add_argument( diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index af7976c30..4e7eea852 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -114,7 +114,7 @@ class Command(BaseCommand): Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) {TerminalColors.ENDC} - """ # noqa + """ # noqa ) if len(skipped_domain_entries) > 0: logger.info( @@ -192,16 +192,15 @@ class Command(BaseCommand): # ---- UPDATE THE DOMAIN # update the status - self.update_domain_status( - transition_domain, target_domain, debug_on - ) - # TODO: not all domains need to be updated (the information is the same). + self.update_domain_status(transition_domain, target_domain, debug_on) + # TODO: not all domains need to be updated + # (the information is the same). # Need to bubble this up to the final report. # update dates (creation and expiration) if transition_domain_creation_date is not None: - # TODO: added this because I ran into a situation where - # the created_at date was null (violated a key constraint). + # TODO: added this because I ran into a situation where + # the created_at date was null (violated a key constraint). # How do we want to handle this case? target_domain.created_at = transition_domain_creation_date @@ -493,7 +492,9 @@ class Command(BaseCommand): self.update_domain_information( target_domain_information, template_domain_information, debug_on ) - # TODO: not all domains need to be updated (the information is the same). Need to bubble this up to the final report. + # TODO: not all domains need to be updated + # (the information is the same). + # Need to bubble this up to the final report. return (target_domain_information, domain, False) except DomainInformation.MultipleObjectsReturned: @@ -594,8 +595,11 @@ class Command(BaseCommand): debug_on, f"{TerminalColors.OKCYAN}" "Processing Transition Domain: " - f"{transition_domain_name}, {transition_domain_status}, {transition_domain_email}" - f", {transition_domain_creation_date}, {transition_domain_expiration_date}" + f"{transition_domain_name}," + f" {transition_domain_status}," + f" {transition_domain_email}" + f", {transition_domain_creation_date}, " + f"{transition_domain_expiration_date}" f"{TerminalColors.ENDC}", # noqa ) @@ -680,7 +684,8 @@ class Command(BaseCommand): else: # Raise an err for now raise Exception( - f"Domain {existing_domain} wants to be added but doesn't exist in the DB" + f"Domain {existing_domain} wants to be added" + "but doesn't exist in the DB" ) invitation.save() @@ -740,7 +745,9 @@ class Command(BaseCommand): ), None, ) - # TODO: this is redundant. Currently debugging....running into unique key constraint error.... + # TODO: this is redundant. + # Currently debugging.... + # running into unique key constraint error.... existing_domain_info = DomainInformation.objects.filter( domain__name=target_domain_information.domain.name ).exists() @@ -765,7 +772,8 @@ class Command(BaseCommand): f"updated domain information: {target_domain_information}" ) else: - debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}" + debug_string = "domain information already exists and " + f"matches incoming data (NO CHANGES MADE): {target_domain_information}" # DEBUG: TerminalHelper.print_conditional( diff --git a/src/registrar/management/commands/utility/epp_data_containers.py b/src/registrar/management/commands/utility/epp_data_containers.py index 3fe170574..1f370dca7 100644 --- a/src/registrar/management/commands/utility/epp_data_containers.py +++ b/src/registrar/management/commands/utility/epp_data_containers.py @@ -4,7 +4,7 @@ A list of helper classes to facilitate handling data from verisign data exports. Regarding our dataclasses: Not intended to be used as models but rather as an alternative to storing as a dictionary. By keeping it as a dataclass instead of a dictionary, we can maintain data consistency. -""" +""" # noqa from dataclasses import dataclass, field from datetime import date from enum import Enum diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index b06bc5299..e7e1ae745 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -239,6 +239,7 @@ class LoadExtraTransitionDomain: total_transition_domains = len(updated_transition_domains) total_updates_made = TransitionDomain.objects.all().count() if total_transition_domains != total_updates_made: + # noqa here for line length logger.error( f"""{TerminalColors.FAIL} WARNING: something went wrong processing domain information data. @@ -251,7 +252,7 @@ class LoadExtraTransitionDomain: corrupt data. Please check logs to diagnose. ----- TERMINATING ---- - """ + """ # noqa ) sys.exit() @@ -654,7 +655,7 @@ class FileDataHolder: id_field: data_type(...), ... } - """ + """ # noqa def __init__( self, @@ -664,18 +665,18 @@ class FileDataHolder: id_field: str, ): # Metadata # - ## Filename inference metadata ## + # = Filename inference metadata =# self.regex = regex self.could_infer = False - ## "data" object metadata ## - ### Where the data is sourced from ### + # = "data" object metadata =# + # == Where the data is sourced from ==# self.filename = filename - ### What type the data is ### + # == What type the data is ==# self.data_type = data_type - ### What the id should be in the holding dict ### + # == What the id should be in the holding dict ==# # TODO - rename to id_field_name self.id_field = id_field @@ -868,8 +869,10 @@ class ExtraTransitionDomain: continue # Infer filename logic # - # This mode is used for internal development use and testing only. Rather than having - # to manually define the filename each time, we can infer what the filename + # This mode is used for + # internal development use and testing only. + # Rather than havingto manually define the + # filename each time, we can infer what the filename # actually is. # Not intended for use outside of that, as it is better to assume diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 05c37b271..1d6fb23a5 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -180,7 +180,7 @@ class TerminalHelper: # If the user decided to proceed return true. # Otherwise, either return false or exit this subroutine. - if proceed_execution == False: + if not proceed_execution: if system_exit_on_terminate: sys.exit() return False @@ -206,7 +206,9 @@ class TerminalHelper: filepath = f"{file_directory}{filename}.txt" # Write to file logger.info( - f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}" + f"{TerminalColors.MAGENTA}Writing to file " + f" {filepath}..." + f"{TerminalColors.ENDC}" ) with open(f"{filepath}", "w+") as f: f.write(file_contents) @@ -233,7 +235,7 @@ class TerminalHelper: length - Optional : character length of bar (Int) fill - Optional : bar fill character (Str) printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) - """ + """ # noqa """ # Initial call to print 0% progress @@ -243,7 +245,7 @@ class TerminalHelper: time.sleep(0.1) # Update Progress Bar printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) - """ + """ # noqa percent = ("{0:." + str(decimals) + "f}").format( 100 * (iteration / float(total)) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 11f3afe7c..3a31fb8e7 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -36,7 +36,7 @@ class TransitionDomainArguments: limitParse: Optional[int] = field(default=None, repr=True) # Filenames # - ## Adhocs ## + # = Adhocs =# agency_adhoc_filename: Optional[str] = field( default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True ) @@ -50,7 +50,7 @@ class TransitionDomainArguments: default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True ) - ## Data files ## + # = Data files =# domain_escrow_filename: Optional[str] = field( default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True ) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index fb93e52b6..0a16d5359 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -47,8 +47,10 @@ class TestMigrations(TestCase): UserDomainRole.objects.all().delete() def run_load_domains(self): + # noqa here because splitting this up makes it confusing. + # ES501 with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa return_value=True, ): call_command( @@ -61,8 +63,10 @@ class TestMigrations(TestCase): call_command("transfer_transition_domains_to_domains") def run_master_script(self): + # noqa here (E501) because splitting this up makes it + # confusing to read. with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa return_value=True, ): call_command( From 733ddae9987074556f93ed7412f4ff8d759c5434 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:28:22 -0700 Subject: [PATCH 74/88] Fix linting issues --- .../transfer_transition_domains_to_domains.py | 315 +++++++++++------- 1 file changed, 191 insertions(+), 124 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 4e7eea852..81e5ed041 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -531,57 +531,117 @@ class Command(BaseCommand): ) return (target_domain_information, domain, True) - # ====================================================== - # ===================== HANDLE ======================== - # ====================================================== - def handle( + # C901 'Command.handle' is too complex + def process_domain_information( self, - **options, + valid_agency_choices, + valid_fed_choices, + valid_org_choices, + debug_on, + skipped_domain_information_entries, + domain_information_to_create, + updated_domain_information, + debug_max_entries_to_parse, + total_rows_parsed, ): - """Parse entries in TransitionDomain table - and create (or update) corresponding entries in the - Domain and DomainInvitation tables.""" + for transition_domain in TransitionDomain.objects.all(): + ( + target_domain_information, + associated_domain, + was_created, + ) = self.update_or_create_domain_information( + transition_domain, + valid_agency_choices, + valid_fed_choices, + valid_org_choices, + debug_on, + ) - # grab command line arguments and store locally... - debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) # set to 0 to parse all entries + debug_string = "" + if target_domain_information is None: + # ---------------- SKIPPED ---------------- + skipped_domain_information_entries.append(target_domain_information) + debug_string = ( + f"skipped domain information: {target_domain_information}" + ) + elif was_created: + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + ( + f"{TerminalColors.OKCYAN}" + f"Checking duplicates for: {target_domain_information}" + f"{TerminalColors.ENDC}" + ), # noqa + ) + # ---------------- DUPLICATE ---------------- + # The unique key constraint does not allow multiple domain + # information objects to share the same domain + existing_domain_information_in_to_create = next( + ( + x + for x in domain_information_to_create + if x.domain.name == target_domain_information.domain.name + ), + None, + ) + # TODO: this is redundant. + # Currently debugging.... + # running into unique key constraint error.... + existing_domain_info = DomainInformation.objects.filter( + domain__name=target_domain_information.domain.name + ).exists() + if ( + existing_domain_information_in_to_create is not None + or existing_domain_info + ): + debug_string = f"""{TerminalColors.YELLOW} + Duplicate Detected: {existing_domain_information_in_to_create}. + Cannot add duplicate Domain Information object + {TerminalColors.ENDC}""" + else: + # ---------------- CREATED ---------------- + domain_information_to_create.append(target_domain_information) + debug_string = ( + f"created domain information: {target_domain_information}" + ) + elif not was_created: + # ---------------- UPDATED ---------------- + updated_domain_information.append(target_domain_information) + debug_string = ( + f"updated domain information: {target_domain_information}" + ) + else: + debug_string = "domain information already exists and " + f"matches incoming data (NO CHANGES MADE): {target_domain_information}" - self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN}{debug_string}{TerminalColors.ENDC}"), + ) - # domains to ADD - domains_to_create = [] - domain_information_to_create = [] - - # domains we UPDATED - updated_domain_entries = [] - updated_domain_information = [] - - # domains we SKIPPED - skipped_domain_entries = [] - skipped_domain_information_entries = [] - - # domain invitations to ADD - domain_invitations_to_create = [] - - # if we are limiting our parse (for testing purposes, keep - # track of total rows parsed) - total_rows_parsed = 0 - - logger.info( - f"""{TerminalColors.OKCYAN} - ========================== - Beginning Data Transfer - ========================== - {TerminalColors.ENDC}""" + # ------------------ Parse limit reached? ------------------ + # Check parse limit and exit loop if parse limit has been reached + if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): + break + return ( + skipped_domain_information_entries, + domain_information_to_create, + updated_domain_information, ) - logger.info( - f"""{TerminalColors.OKCYAN} - ========= Adding Domains and Domain Invitations ========= - {TerminalColors.ENDC}""" - ) + # C901 'Command.handle' is too complex + def process_domain_and_invitations( + self, + debug_on, + skipped_domain_entries, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + debug_max_entries_to_parse, + total_rows_parsed, + ): for transition_domain in TransitionDomain.objects.all(): # Create some local variables to make data tracing easier transition_domain_name = transition_domain.domain_name @@ -665,6 +725,78 @@ class Command(BaseCommand): # Check parse limit and exit loop if parse limit has been reached if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): break + return ( + skipped_domain_entries, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + ) + + # ====================================================== + # ===================== HANDLE ======================== + # ====================================================== + def handle( + self, + **options, + ): + """Parse entries in TransitionDomain table + and create (or update) corresponding entries in the + Domain and DomainInvitation tables.""" + + # grab command line arguments and store locally... + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + + self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + + # domains to ADD + domains_to_create = [] + domain_information_to_create = [] + + # domains we UPDATED + updated_domain_entries = [] + updated_domain_information = [] + + # domains we SKIPPED + skipped_domain_entries = [] + skipped_domain_information_entries = [] + + # domain invitations to ADD + domain_invitations_to_create = [] + + # if we are limiting our parse (for testing purposes, keep + # track of total rows parsed) + total_rows_parsed = 0 + + logger.info( + f"""{TerminalColors.OKCYAN} + ========================== + Beginning Data Transfer + ========================== + {TerminalColors.ENDC}""" + ) + + logger.info( + f"""{TerminalColors.OKCYAN} + ========= Adding Domains and Domain Invitations ========= + {TerminalColors.ENDC}""" + ) + ( + skipped_domain_entries, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + ) = self.process_domain_and_invitations( + debug_on, + skipped_domain_entries, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + debug_max_entries_to_parse, + total_rows_parsed, + ) # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) @@ -704,87 +836,22 @@ class Command(BaseCommand): ========= Adding Domains Information Objects ========= {TerminalColors.ENDC}""" ) - for transition_domain in TransitionDomain.objects.all(): - ( - target_domain_information, - associated_domain, - was_created, - ) = self.update_or_create_domain_information( - transition_domain, - valid_agency_choices, - valid_fed_choices, - valid_org_choices, - debug_on, - ) - debug_string = "" - if target_domain_information is None: - # ---------------- SKIPPED ---------------- - skipped_domain_information_entries.append(target_domain_information) - debug_string = ( - f"skipped domain information: {target_domain_information}" - ) - elif was_created: - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - ( - f"{TerminalColors.OKCYAN}" - f"Checking duplicates for: {target_domain_information}" - f"{TerminalColors.ENDC}" - ), # noqa - ) - # ---------------- DUPLICATE ---------------- - # The unique key constraint does not allow multiple domain - # information objects to share the same domain - existing_domain_information_in_to_create = next( - ( - x - for x in domain_information_to_create - if x.domain.name == target_domain_information.domain.name - ), - None, - ) - # TODO: this is redundant. - # Currently debugging.... - # running into unique key constraint error.... - existing_domain_info = DomainInformation.objects.filter( - domain__name=target_domain_information.domain.name - ).exists() - if ( - existing_domain_information_in_to_create is not None - or existing_domain_info - ): - debug_string = f"""{TerminalColors.YELLOW} - Duplicate Detected: {existing_domain_information_in_to_create}. - Cannot add duplicate Domain Information object - {TerminalColors.ENDC}""" - else: - # ---------------- CREATED ---------------- - domain_information_to_create.append(target_domain_information) - debug_string = ( - f"created domain information: {target_domain_information}" - ) - elif not was_created: - # ---------------- UPDATED ---------------- - updated_domain_information.append(target_domain_information) - debug_string = ( - f"updated domain information: {target_domain_information}" - ) - else: - debug_string = "domain information already exists and " - f"matches incoming data (NO CHANGES MADE): {target_domain_information}" - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - (f"{TerminalColors.OKCYAN}{debug_string}{TerminalColors.ENDC}"), - ) - - # ------------------ Parse limit reached? ------------------ - # Check parse limit and exit loop if parse limit has been reached - if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): - break + ( + skipped_domain_information_entries, + domain_information_to_create, + updated_domain_information, + ) = self.process_domain_information( + valid_agency_choices, + valid_fed_choices, + valid_org_choices, + debug_on, + skipped_domain_information_entries, + domain_information_to_create, + updated_domain_information, + debug_max_entries_to_parse, + total_rows_parsed, + ) TerminalHelper.print_conditional( debug_on, From 26bf6dc73c8f8cdd1c10c829d135402933bd42c9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 10:45:22 -0700 Subject: [PATCH 75/88] Linting --- .../transfer_transition_domains_to_domains.py | 9 ++-- .../utility/extra_transition_domain_helper.py | 44 +++++++++++++++---- .../commands/utility/terminal_helper.py | 7 +-- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 81e5ed041..82e94ef2d 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -254,7 +254,7 @@ class Command(BaseCommand): # ----------------------- CREATE DOMAIN ----------------------- # no matching entry, make one target_domain = Domain( - name=transition_domain_name, + name=str(transition_domain_name), state=transition_domain_status, expiration_date=transition_domain_expiration_date, ) @@ -377,11 +377,12 @@ class Command(BaseCommand): org_choices, debug_on, ) -> DomainInformation: - org_type = transition_domain.organization_type + org_type_current = transition_domain.organization_type fed_type = transition_domain.federal_type fed_agency = transition_domain.federal_agency - match org_type: + org_type = ("", "") + match org_type_current: case "Federal": org_type = ("federal", "Federal") case "Interstate": @@ -414,7 +415,7 @@ class Command(BaseCommand): elif debug_on: logger.debug(f"No org type found on {domain.name}") - if valid_fed_type: + if valid_fed_type and isinstance(fed_type, str): new_domain_info_data["federal_type"] = fed_type.lower() elif debug_on: logger.debug(f"No federal type found on {domain.name}") diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index e7e1ae745..b68b9dc27 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -9,7 +9,7 @@ import logging import os import sys -from typing import List, Tuple +from typing import Dict, List, Tuple from registrar.models.transition_domain import TransitionDomain @@ -321,7 +321,10 @@ class LoadExtraTransitionDomain: and transition_domain.federal_agency.strip() != "" ) - if not info.active.lower() == "y": + if ( + not isinstance(info.active, str) or + not info.active.lower() == "y" + ): self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -331,7 +334,10 @@ class LoadExtraTransitionDomain: ) return transition_domain - if not info.isfederal.lower() == "y": + if ( + not isinstance(info.isfederal, str) or + not info.isfederal.lower() == "y" + ): self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -377,7 +383,9 @@ class LoadExtraTransitionDomain: # This data is stored as follows: FEDERAL - Judicial # For all other records, it is stored as so: Interstate # We can infer if it is federal or not based on this fact. - domain_type = info.domaintype.split("-") + domain_type = [] + if isinstance(info.domaintype, str): + domain_type = info.domaintype.split("-") domain_type_length = len(domain_type) if domain_type_length < 1 or domain_type_length > 2: raise ValueError("Found invalid data on DOMAIN_ADHOC") @@ -387,7 +395,10 @@ class LoadExtraTransitionDomain: # Check if this domain_type is active or not. # If not, we don't want to add this. - if not info.active.lower() == "y": + if ( + not isinstance(info.active, str) or + not info.active.lower() == "y" + ): self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -681,7 +692,7 @@ class FileDataHolder: self.id_field = id_field # Object data # - self.data = {} + self.data: Dict[str, type] = {} def try_infer_filename(self, current_file_name, default_file_name): """Tries to match a given filename to a regex, @@ -798,7 +809,7 @@ class ExtraTransitionDomain: # TODO - revise comment def populate_file_data( - self, pattern_map_params: List[Tuple[EnumFilenames, str, type, str]] + self, pattern_map_params ): """Populates the self.file_data field given a set of tuple params. @@ -935,6 +946,21 @@ class ExtraTransitionDomain: ) return dict_data + def _grab_row_id(self, row, id_field, file, dataclass_type): + try: + row_id = row[id_field] + except KeyError as err: + logger.error( + f"{TerminalColors.FAIL}" + "\n Key mismatch! Did you upload the wrong file?" + f"\n File: {file}" + f"\n Expected type: {dataclass_type}" + f"{TerminalColors.ENDC}" + ) + raise err + else: + return row_id + def _read_csv_file(self, file, seperator, dataclass_type, id_field): dict_data = {} # Used when we encounter bad data @@ -955,8 +981,8 @@ class ExtraTransitionDomain: ) dict_data = {} break - - row_id = row[id_field] + + row_id = self._grab_row_id(row, id_field, file, dataclass_type) # To maintain pairity with the load_transition_domain # script, we store this data in lowercase. diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 1d6fb23a5..24e7147ce 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -1,6 +1,7 @@ from enum import Enum import logging import sys +from typing import List logger = logging.getLogger(__name__) @@ -42,7 +43,7 @@ class TerminalColors: class TerminalHelper: @staticmethod - def query_yes_no(question: str, default="yes") -> bool: + def query_yes_no(question: str, default="yes"): """Ask a yes/no question via raw_input() and return their answer. "question" is a string that is presented to the user. @@ -73,7 +74,7 @@ class TerminalHelper: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") @staticmethod - def query_yes_no_exit(question: str, default="yes") -> bool: + def query_yes_no_exit(question: str, default="yes"): """Ask a yes/no question via raw_input() and return their answer. "question" is a string that is presented to the user. @@ -113,7 +114,7 @@ class TerminalHelper: logger.info("Please respond with a valid selection.\n") # @staticmethod - def array_as_string(array_to_convert: []) -> str: + def array_as_string(array_to_convert: List[str]) -> str: array_as_string = "{}".format("\n".join(map(str, array_to_convert))) return array_as_string From 029582925fbb2da0af4b7b837c9cab66dd034960 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:19:36 -0700 Subject: [PATCH 76/88] Update data_migration.md --- docs/operations/data_migration.md | 44 +++++++++++++------------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index a77266282..e874a2d83 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -86,12 +86,13 @@ We are provided with information about Transition Domains in the following files - FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain names to contact ID. Domains in this file will usually have 3 contacts each - FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) - FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses -- FILE 4: **escrow_domains.daily.dotgov.GOV.txt** -> has basic domain data +- FILE 4: **escrow_domains.daily.dotgov.GOV.txt** -> has a map of domainname, expiration and creation dates - FILE 5: **domainadditionaldatalink.adhoc.dotgov.txt** -> has the map of domains to other data like authority, organization, & domain type -- FILE 6: **domaintypes.adhoc.dotgov.txt** -> has domain type data -- FILE 7: **organization.adhoc.dotgov.txt** -> has organization data -- FILE 8: **authority.adhoc.dotgov.txt** -> has authority data -- FILE 9: **agency.adhoc.dotgov.txt** -> has agency data +- FILE 6: **domaintypes.adhoc.dotgov.txt** -> has data on federal type and organization type +- FILE 7: **organization.adhoc.dotgov.txt** -> has organization name data +- FILE 8: **authority.adhoc.dotgov.txt** -> has authority data which maps to an agency +- FILE 9: **agency.adhoc.dotgov.txt** -> has federal agency data +- FILE 10: **migrationFilepaths.json** -> A JSON which points towards all given filenames. Specified below. #### STEP 2: obtain JSON file (for file locations) Add a JSON file called "migrationFilepaths.json" with the following contents (update filenames and directory as needed): @@ -110,9 +111,18 @@ Add a JSON file called "migrationFilepaths.json" with the following contents (up } ``` +This JSON file can exist anywhere, but to keep things simple, add it to the same folder as used in step 1. `src/migrationdata`. +Directory specifies the directory that the given `filenames` exist in. For instance, a `contacts_filename` of `test.txt` with a `directory` of `migrationdata` would need to exist under `migrationdata/test.txt`. + We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. +#### STEP 3: Bundle all relevant data files into one file +Move all the files specified in Step 1 into a shared folder, and create a tar.gz + +Create a folder on your desktop called `datafiles` and move all of the obtained files into that. Add these files to a tar.gz archive using any method. See (here)[https://stackoverflow.com/questions/53283240/how-to-create-tar-file-with-7zip]. + + ### SECTION 1 - SANDBOX MIGRATION SETUP Load migration data onto a production or sandbox environment @@ -136,8 +146,6 @@ cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap CloudFoundry supports scp as means of transferring data locally to our environment. If you are dealing with a batch of files, try sending across a tar.gz and unpacking that. - - ##### Login to Cloud.gov ```bash @@ -208,7 +216,6 @@ tar -xvf migrationdata/{FILE_NAME}.tar.gz -C migrationdata/ --strip-components=1 *FILE_NAME* - Name of the desired file, ex: exportdata - #### Manual method If the `cat_files_into_getgov.py` script isn't working, follow these steps instead. @@ -235,27 +242,12 @@ This will allow Docker to mount the files to a container (under `/app`) for our *You are now ready to run migration scripts.* ## Transition Domains (Part 2) - Running the Migration Scripts - +While keeping the same ssh instance open (if you are running on a sandbox), run through the following commands. If you run into the error that. If you cannot run `manage.py` commands, try running `/tmp/lifecycle/shell` in the ssh instance. ### STEP 1: Load Transition Domains -Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. -##### Create a JSON file -In your chosen directory (either `src/tmp` or `src/migrationdata` depending on preference), create a json file called `migrationFilepaths.json`. This file will map to other urls -Example -``` -{ - "directory": "migrationdata/", - "agency_adhoc_filename": "20231009.agency.adhoc.dotgov.txt", - "authority_adhoc_filename": "authority.adhoc.dotgov.txt", - "contacts_filename": "escrow_contacts.daily.dotgov.GOV.txt", - "domain_adhoc_filename": "20231009.domaintypes.adhoc.dotgov.txt", - "domain_additional_filename": "20231009.domainadditionaldatalink.adhoc.dotgov.txt", - "domain_contacts_filename": "escrow_domain_contacts.daily.dotgov.GOV.txt", - "domain_escrow_filename": "escrow_domains.daily.dotgov.GOV.txt", - "domain_statuses_filename": "escrow_domain_statuses.daily.dotgov.GOV.txt", - "organization_adhoc_filename": "20231009.organization.adhoc.dotgov.txt" -} +Run the following command, making sure the file paths point to the right location. This will parse all given files and load the information into the TransitionDomain table. Make sure you have your migrationFilepaths.json file in the same directory. + ``` ##### LOCAL COMMAND ```shell From 7afb6a3e36bd50803b5dea81017055662681046e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:20:07 -0700 Subject: [PATCH 77/88] Update mixins.py --- src/registrar/views/utility/mixins.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 0db1ff402..02cc25d64 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -100,13 +100,7 @@ class DomainPermission(PermissionsLoginMixin): if DomainInformation.objects.filter(id=pk).exists(): requested_domain = DomainInformation.objects.get(id=pk) - # If no domain_application object exists and we are - # coming from the manage_domain dashboard, this is likely - # a transition domain. domain_application = requested_domain.domain_application - if not hasattr(domain_application, "status"): - return True - if domain_application.status not in valid_domain_statuses: return False From fef39c4cbea584d26554b28ae208d4b602ec0d09 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:33:30 -0700 Subject: [PATCH 78/88] Fix migrations --- .../utility/extra_transition_domain_helper.py | 23 +++++-------------- ...ansitiondomain_federal_agency_and_more.py} | 2 +- 2 files changed, 7 insertions(+), 18 deletions(-) rename src/registrar/migrations/{0044_transitiondomain_federal_agency_and_more.py => 0045_transitiondomain_federal_agency_and_more.py} (97%) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index b68b9dc27..89e50742e 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -9,7 +9,7 @@ import logging import os import sys -from typing import Dict, List, Tuple +from typing import Dict from registrar.models.transition_domain import TransitionDomain @@ -321,10 +321,7 @@ class LoadExtraTransitionDomain: and transition_domain.federal_agency.strip() != "" ) - if ( - not isinstance(info.active, str) or - not info.active.lower() == "y" - ): + if not isinstance(info.active, str) or not info.active.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -334,10 +331,7 @@ class LoadExtraTransitionDomain: ) return transition_domain - if ( - not isinstance(info.isfederal, str) or - not info.isfederal.lower() == "y" - ): + if not isinstance(info.isfederal, str) or not info.isfederal.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -395,10 +389,7 @@ class LoadExtraTransitionDomain: # Check if this domain_type is active or not. # If not, we don't want to add this. - if ( - not isinstance(info.active, str) or - not info.active.lower() == "y" - ): + if not isinstance(info.active, str) or not info.active.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, LogCode.ERROR, @@ -808,9 +799,7 @@ class ExtraTransitionDomain: self.file_data = self.populate_file_data(pattern_map_params) # TODO - revise comment - def populate_file_data( - self, pattern_map_params - ): + def populate_file_data(self, pattern_map_params): """Populates the self.file_data field given a set of tuple params. @@ -981,7 +970,7 @@ class ExtraTransitionDomain: ) dict_data = {} break - + row_id = self._grab_row_id(row, id_field, file, dataclass_type) # To maintain pairity with the load_transition_domain diff --git a/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py b/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py similarity index 97% rename from src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py rename to src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py index 838a26cd4..742ab4453 100644 --- a/src/registrar/migrations/0044_transitiondomain_federal_agency_and_more.py +++ b/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py @@ -5,7 +5,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("registrar", "0043_domain_expiration_date"), + ("registrar", "0044_create_groups_v04"), ] operations = [ From b6f3b7a1e01aed2f6d567d78e8ab5dd41f9e4898 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:42:38 -0700 Subject: [PATCH 79/88] Final lint pass --- src/registrar/management/commands/load_transition_domain.py | 1 - .../commands/transfer_transition_domains_to_domains.py | 5 +++-- src/registrar/management/commands/utility/terminal_helper.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 20c0c34e1..4b48298ae 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -460,7 +460,6 @@ class Command(BaseCommand): logger.info("Reading domain-contacts data file %s", domain_contacts_filename) with open(domain_contacts_filename, "r") as domain_contacts_file: for row in csv.reader(domain_contacts_file, delimiter=sep): - # TerminalHelper.printProgressBar(total_rows_parsed, total_lines) total_rows_parsed += 1 # fields are just domain, userid, role diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 82e94ef2d..ca9995407 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -1,6 +1,7 @@ import logging import argparse import sys +from typing import Tuple from django_fsm import TransitionNotAllowed # type: ignore @@ -161,7 +162,7 @@ class Command(BaseCommand): # ====================================================== def update_or_create_domain( self, transition_domain: TransitionDomain, debug_on: bool - ) -> (Domain, bool): + ) -> Tuple[Domain, bool]: """Given a transition domain, either finds & updates an existing corresponding domain, or creates a new corresponding domain in the Domain table. @@ -445,7 +446,7 @@ class Command(BaseCommand): fed_choices, org_choices, debug_on: bool, - ) -> (DomainInformation, bool): + ) -> Tuple[DomainInformation, bool]: transition_domain_name = transition_domain.domain_name # Get associated domain diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 24e7147ce..b38176172 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -113,7 +113,7 @@ class TerminalHelper: else: logger.info("Please respond with a valid selection.\n") - # @staticmethod + @staticmethod def array_as_string(array_to_convert: List[str]) -> str: array_as_string = "{}".format("\n".join(map(str, array_to_convert))) return array_as_string From 59cd35469492799633bd97bb1b01b26b91be0912 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:47:45 -0700 Subject: [PATCH 80/88] Update transfer_transition_domains_to_domains.py --- .../commands/transfer_transition_domains_to_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index ca9995407..0d31e8411 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -446,7 +446,7 @@ class Command(BaseCommand): fed_choices, org_choices, debug_on: bool, - ) -> Tuple[DomainInformation, bool]: + ): transition_domain_name = transition_domain.domain_name # Get associated domain From 6c405d7ba1585ba531a5e575286c868e7d7dc3fa Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:51:17 -0700 Subject: [PATCH 81/88] Update transfer_transition_domains_to_domains.py --- .../commands/transfer_transition_domains_to_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 0d31e8411..6482401bd 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -162,7 +162,7 @@ class Command(BaseCommand): # ====================================================== def update_or_create_domain( self, transition_domain: TransitionDomain, debug_on: bool - ) -> Tuple[Domain, bool]: + ): """Given a transition domain, either finds & updates an existing corresponding domain, or creates a new corresponding domain in the Domain table. From 639202b72c560fd47c1c3ae466eaa4d702920c8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:54:12 -0700 Subject: [PATCH 82/88] Linting final --- .../commands/transfer_transition_domains_to_domains.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 6482401bd..0b64788cd 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -1,7 +1,6 @@ import logging import argparse import sys -from typing import Tuple from django_fsm import TransitionNotAllowed # type: ignore From cfe177612a06c0e35048625f86c82392e2e8d28f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:04:06 -0700 Subject: [PATCH 83/88] Add more specificity in the readme --- docs/operations/data_migration.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index e874a2d83..3f97511f3 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -114,14 +114,18 @@ Add a JSON file called "migrationFilepaths.json" with the following contents (up This JSON file can exist anywhere, but to keep things simple, add it to the same folder as used in step 1. `src/migrationdata`. Directory specifies the directory that the given `filenames` exist in. For instance, a `contacts_filename` of `test.txt` with a `directory` of `migrationdata` would need to exist under `migrationdata/test.txt`. +Later on, we will bundle this file along with the others into its own folder. Keep it within the `migrationdata/` directory if you are passing data to your sandbox, for simplicity. + We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. -#### STEP 3: Bundle all relevant data files into one file -Move all the files specified in Step 1 into a shared folder, and create a tar.gz +#### STEP 3: Bundle all relevant data files into an archive +Move all the files specified in Step 1 into a shared folder, and create a tar.gz. Create a folder on your desktop called `datafiles` and move all of the obtained files into that. Add these files to a tar.gz archive using any method. See (here)[https://stackoverflow.com/questions/53283240/how-to-create-tar-file-with-7zip]. +After this is created, move this archive into `src/migrationdata`. + ### SECTION 1 - SANDBOX MIGRATION SETUP Load migration data onto a production or sandbox environment From 75f4e7f4b17ef0c475bd25768e51537627641c02 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:30:08 -0700 Subject: [PATCH 84/88] PR changes --- docs/operations/data_migration.md | 2 +- .../commands/load_transition_domain.py | 19 +++++--- .../utility/extra_transition_domain_helper.py | 10 +++-- .../commands/utility/terminal_helper.py | 44 ------------------- src/registrar/models/domain_information.py | 1 + 5 files changed, 21 insertions(+), 55 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 3f97511f3..38411e372 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -246,7 +246,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our *You are now ready to run migration scripts.* ## Transition Domains (Part 2) - Running the Migration Scripts -While keeping the same ssh instance open (if you are running on a sandbox), run through the following commands. If you run into the error that. If you cannot run `manage.py` commands, try running `/tmp/lifecycle/shell` in the ssh instance. +While keeping the same ssh instance open (if you are running on a sandbox), run through the following commands.If you cannot run `manage.py` commands, try running `/tmp/lifecycle/shell` in the ssh instance. ### STEP 1: Load Transition Domains diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 4b48298ae..79fe83022 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -5,6 +5,7 @@ import logging import argparse from collections import defaultdict +from django.conf import settings from django.core.management import BaseCommand from registrar.management.commands.utility.epp_data_containers import EnumFilenames @@ -63,12 +64,15 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - parser.add_argument( - "--infer_filenames", - action=argparse.BooleanOptionalAction, - help="Determines if we should infer filenames or not." - "Recommended to be enabled only in a development or testing setting.", - ) + # This option should only be available when developing locally. + # This should not be available to the end user. + if settings.DEBUG: + parser.add_argument( + "--infer_filenames", + action=argparse.BooleanOptionalAction, + help="Determines if we should infer filenames or not." + "Recommended to be enabled only in a development or testing setting.", + ) parser.add_argument( "--directory", default="migrationdata", help="Desired directory" @@ -322,6 +326,9 @@ class Command(BaseCommand): **options, ): """Parse the data files and create TransitionDomains.""" + if not settings.DEBUG: + options["infer_filenames"] = False + args = TransitionDomainArguments(**options) # Desired directory for additional TransitionDomain data diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 89e50742e..eb73125e1 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -334,12 +334,11 @@ class LoadExtraTransitionDomain: if not isinstance(info.isfederal, str) or not info.isfederal.lower() == "y": self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ADHOC, - LogCode.ERROR, - f"Could not add non-federal agency {info.agencyname} on {domain_name}", + LogCode.INFO, + f"Adding non-federal agency {info.agencyname} on {domain_name}", domain_name, not self.debug, ) - return transition_domain transition_domain.federal_agency = info.agencyname @@ -685,6 +684,10 @@ class FileDataHolder: # Object data # self.data: Dict[str, type] = {} + # This is used ONLY for development purposes. This behaviour + # is controlled by the --infer_filename flag which is defaulted + # to false. The purpose of this check is to speed up development, + # but it cannot be used by the enduser def try_infer_filename(self, current_file_name, default_file_name): """Tries to match a given filename to a regex, then uses that match to generate the filename.""" @@ -850,7 +853,6 @@ class ExtraTransitionDomain: infer_filenames: bool -> Determines if we should try to infer the filename if a default is passed in """ - self.clear_file_data() for name, value in self.file_data.items(): is_domain_escrow = name == EnumFilenames.DOMAIN_ESCROW filename = f"{value.filename}" diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index b38176172..56e0b2bc6 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -213,47 +213,3 @@ class TerminalHelper: ) with open(f"{filepath}", "w+") as f: f.write(file_contents) - - @staticmethod - def printProgressBar( - iteration, - total, - prefix="Progress:", - suffix="Complete", - decimals=1, - length=100, - fill="█", - printEnd="\r", - ): - """ - Call in a loop to create terminal progress bar - @params: - iteration - Required : current iteration (Int) - total - Required : total iterations (Int) - prefix - Optional : prefix string (Str) - suffix - Optional : suffix string (Str) - decimals - Optional : positive number of decimals in percent complete (Int) - length - Optional : character length of bar (Int) - fill - Optional : bar fill character (Str) - printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) - """ # noqa - - """ - # Initial call to print 0% progress - printProgressBar(0, l, prefix = 'Progress:', suffix = 'Complete', length = 50) - for i, item in enumerate(items): - # Do stuff... - time.sleep(0.1) - # Update Progress Bar - printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) - """ # noqa - - percent = ("{0:." + str(decimals) + "f}").format( - 100 * (iteration / float(total)) - ) - filledLength = int(length * iteration // total) - bar = fill * filledLength + "-" * (length - filledLength) - print(f"\r{prefix} |{bar}| {percent}% {suffix}", end=printEnd) - # Print New Line on Complete - if iteration == total: - print() diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 377d75685..73d724815 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -54,6 +54,7 @@ class DomainInformation(TimeStampedModel): blank=True, help_text="Type of Organization", ) + federally_recognized_tribe = models.BooleanField( null=True, help_text="Is the tribe federally recognized", From c8db49b1fbd7a7c342527bab4e9d785b291f18a4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:33:15 -0700 Subject: [PATCH 85/88] Update domain_information.py --- src/registrar/models/domain_information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 73d724815..d2bc5c53d 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -54,7 +54,7 @@ class DomainInformation(TimeStampedModel): blank=True, help_text="Type of Organization", ) - + federally_recognized_tribe = models.BooleanField( null=True, help_text="Is the tribe federally recognized", From 1890cc8f26a3c845e2f0f40faae636d90696a168 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:16:51 -0700 Subject: [PATCH 86/88] Add better error messages --- .../commands/load_transition_domain.py | 17 +++++++++++++++++ .../utility/extra_transition_domain_helper.py | 14 ++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 79fe83022..2a92ca8fe 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -1,4 +1,5 @@ import json +import os import sys import csv import logging @@ -426,6 +427,22 @@ class Command(BaseCommand): # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + filenames = [ + agency_adhoc_filename, + domain_adhoc_filename, + organization_adhoc_filename, + domain_escrow_filename, + domain_additional_filename, + ] + + # Do a top-level check to see if these files exist + for filename in filenames: + if not isinstance(filename, str): + raise TypeError(f"Filename must be a string, got {type(filename).__name__}") + full_path = os.path.join(directory, filename) + if not os.path.isfile(full_path): + raise FileNotFoundError(full_path) + # STEP 1: # Create mapping of domain name -> status domain_status_dictionary = self.get_domain_user_dict( diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index eb73125e1..488718d0b 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -867,8 +867,11 @@ class ExtraTransitionDomain: ) else: if not infer_filenames: - logger.error(f"Could not find file: {filename}") - continue + raise FileNotFoundError( + f"{TerminalColors.FAIL}" + f"Could not find file {filename} for {name}" + f"{TerminalColors.ENDC}" + ) # Infer filename logic # # This mode is used for @@ -899,8 +902,11 @@ class ExtraTransitionDomain: is_domain_escrow, ) continue - # Log if we can't find the desired file - logger.error(f"Could not find file: {filename}") + raise FileNotFoundError( + f"{TerminalColors.FAIL}" + f"Could not find file {filename} for {name}" + f"{TerminalColors.ENDC}" + ) def clear_file_data(self): for item in self.file_data.values(): From 8aa840f427ce42f22c80c024bdc1d59d6c8947bb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:31:20 -0700 Subject: [PATCH 87/88] Black linting --- .../commands/load_transition_domain.py | 40 ++------ .../commands/master_domain_migrations.py | 5 +- .../transfer_transition_domains_to_domains.py | 98 +++++-------------- .../utility/extra_transition_domain_helper.py | 93 +++++------------- .../commands/utility/terminal_helper.py | 10 +- .../utility/transition_domain_arguments.py | 24 ++--- ...ransitiondomain_federal_agency_and_more.py | 8 +- src/registrar/models/domain_application.py | 5 +- src/registrar/models/transition_domain.py | 8 +- .../test_transition_domain_migrations.py | 16 +-- 10 files changed, 73 insertions(+), 234 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index b7a249d62..6566a2f16 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -45,10 +45,7 @@ class Command(BaseCommand): """ parser.add_argument( "migration_json_filename", - help=( - "A JSON file that holds the location and filenames" - "of all the data files used for migrations" - ), + help=("A JSON file that holds the location and filenames" "of all the data files used for migrations"), ) parser.add_argument("--sep", default="|", help="Delimiter character") @@ -73,9 +70,7 @@ class Command(BaseCommand): "Recommended to be enabled only in a development or testing setting.", ) - parser.add_argument( - "--directory", default="migrationdata", help="Desired directory" - ) + parser.add_argument("--directory", default="migrationdata", help="Desired directory") parser.add_argument( "--domain_contacts_filename", help="Data file with domain contact information", @@ -119,9 +114,7 @@ class Command(BaseCommand): help="Defines the filename for domain type adhocs", ) - def print_debug_mode_statements( - self, debug_on: bool, debug_max_entries_to_parse: int - ): + def print_debug_mode_statements(self, debug_on: bool, debug_max_entries_to_parse: int): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" if debug_on: @@ -356,42 +349,31 @@ class Command(BaseCommand): debug_on = args.debug # Get --LimitParse argument - debug_max_entries_to_parse = int( - args.limitParse - ) # set to 0 to parse all entries + debug_max_entries_to_parse = int(args.limitParse) # set to 0 to parse all entries # Variables for Additional TransitionDomain Information # # Main script filenames - these do not have defaults domain_contacts_filename = None try: - domain_contacts_filename = directory + options.get( - "domain_contacts_filename" - ) + domain_contacts_filename = directory + options.get("domain_contacts_filename") except TypeError: logger.error( - f"Invalid filename of '{args.domain_contacts_filename}'" - " was provided for domain_contacts_filename" + f"Invalid filename of '{args.domain_contacts_filename}'" " was provided for domain_contacts_filename" ) contacts_filename = None try: contacts_filename = directory + options.get("contacts_filename") except TypeError: - logger.error( - f"Invalid filename of '{args.contacts_filename}'" - " was provided for contacts_filename" - ) + logger.error(f"Invalid filename of '{args.contacts_filename}'" " was provided for contacts_filename") domain_statuses_filename = None try: - domain_statuses_filename = directory + options.get( - "domain_statuses_filename" - ) + domain_statuses_filename = directory + options.get("domain_statuses_filename") except TypeError: logger.error( - f"Invalid filename of '{args.domain_statuses_filename}'" - " was provided for domain_statuses_filename" + f"Invalid filename of '{args.domain_statuses_filename}'" " was provided for domain_statuses_filename" ) # Agency information @@ -630,9 +612,7 @@ class Command(BaseCommand): # Print a summary of findings (duplicate entries, # missing data..etc.) - self.print_summary_duplications( - duplicate_domain_user_combos, duplicate_domains, users_without_email - ) + self.print_summary_duplications(duplicate_domain_user_combos, duplicate_domains, users_without_email) self.print_summary_status_findings(domains_without_status, outlier_statuses) logger.info( diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 36bc2354b..9cb469078 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -94,10 +94,7 @@ class Command(BaseCommand): parser.add_argument( "--migrationJSON", default="migrationFilepaths.json", - help=( - "A JSON file that holds the location and filenames" - "of all the data files used for migrations" - ), + help=("A JSON file that holds the location and filenames" "of all the data files used for migrations"), ) # TODO: deprecate this once JSON module is done? (or keep as an override) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 7c0096720..8e4348db4 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -42,9 +42,7 @@ class Command(BaseCommand): # ====================================================== # ===================== PRINTING ====================== # ====================================================== - def print_debug_mode_statements( - self, debug_on: bool, debug_max_entries_to_parse: int - ): + def print_debug_mode_statements(self, debug_on: bool, debug_max_entries_to_parse: int): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" TerminalHelper.print_conditional( @@ -66,13 +64,8 @@ class Command(BaseCommand): """, ) - def parse_limit_reached( - self, debug_max_entries_to_parse: bool, total_rows_parsed: int - ) -> bool: - if ( - debug_max_entries_to_parse > 0 - and total_rows_parsed >= debug_max_entries_to_parse - ): + def parse_limit_reached(self, debug_max_entries_to_parse: bool, total_rows_parsed: int) -> bool: + if debug_max_entries_to_parse > 0 and total_rows_parsed >= debug_max_entries_to_parse: logger.info( f"""{TerminalColors.YELLOW} ----PARSE LIMIT REACHED. HALTING PARSER.---- @@ -159,9 +152,7 @@ class Command(BaseCommand): # ====================================================== # =================== DOMAIN ===================== # ====================================================== - def update_or_create_domain( - self, transition_domain: TransitionDomain, debug_on: bool - ): + def update_or_create_domain(self, transition_domain: TransitionDomain, debug_on: bool): """Given a transition domain, either finds & updates an existing corresponding domain, or creates a new corresponding domain in the Domain table. @@ -260,9 +251,7 @@ class Command(BaseCommand): ) return (target_domain, True) - def update_domain_status( - self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool - ) -> bool: + def update_domain_status(self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool) -> bool: """Given a transition domain that matches an existing domain, updates the existing domain object with that status of the transition domain. @@ -293,9 +282,7 @@ class Command(BaseCommand): # ====================================================== # ================ DOMAIN INVITATION ================== # ====================================================== - def try_add_domain_invitation( - self, domain_email: str, associated_domain: Domain - ) -> DomainInvitation | None: + def try_add_domain_invitation(self, domain_email: str, associated_domain: Domain) -> DomainInvitation | None: """If no domain invitation exists for the given domain and e-mail, create and return a new domain invitation object. If one already exists, or if the email is invalid, return NONE""" @@ -334,17 +321,11 @@ class Command(BaseCommand): # ====================================================== # ================ DOMAIN INFORMATION ================= # ====================================================== - def update_domain_information( - self, current: DomainInformation, target: DomainInformation, debug_on: bool - ) -> bool: + def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: # DEBUG: TerminalHelper.print_conditional( debug_on, - ( - f"{TerminalColors.OKCYAN}" - f"Updating: {current}" - f"{TerminalColors.ENDC}" - ), # noqa + (f"{TerminalColors.OKCYAN}" f"Updating: {current}" f"{TerminalColors.ENDC}"), # noqa ) updated = False @@ -466,15 +447,11 @@ class Command(BaseCommand): debug_on, ) target_domain_information = None - domain_information_exists = DomainInformation.objects.filter( - domain__name=transition_domain_name - ).exists() + domain_information_exists = DomainInformation.objects.filter(domain__name=transition_domain_name).exists() if domain_information_exists: try: # get the existing domain information object - target_domain_information = DomainInformation.objects.get( - domain__name=transition_domain_name - ) + target_domain_information = DomainInformation.objects.get(domain__name=transition_domain_name) # DEBUG: TerminalHelper.print_conditional( debug_on, @@ -488,9 +465,7 @@ class Command(BaseCommand): # for existing entry, update the status to # the transition domain status - self.update_domain_information( - target_domain_information, template_domain_information, debug_on - ) + self.update_domain_information(target_domain_information, template_domain_information, debug_on) # TODO: not all domains need to be updated # (the information is the same). # Need to bubble this up to the final report. @@ -560,9 +535,7 @@ class Command(BaseCommand): if target_domain_information is None: # ---------------- SKIPPED ---------------- skipped_domain_information_entries.append(target_domain_information) - debug_string = ( - f"skipped domain information: {target_domain_information}" - ) + debug_string = f"skipped domain information: {target_domain_information}" elif was_created: # DEBUG: TerminalHelper.print_conditional( @@ -577,11 +550,7 @@ class Command(BaseCommand): # The unique key constraint does not allow multiple domain # information objects to share the same domain existing_domain_information_in_to_create = next( - ( - x - for x in domain_information_to_create - if x.domain.name == target_domain_information.domain.name - ), + (x for x in domain_information_to_create if x.domain.name == target_domain_information.domain.name), None, ) # TODO: this is redundant. @@ -590,10 +559,7 @@ class Command(BaseCommand): existing_domain_info = DomainInformation.objects.filter( domain__name=target_domain_information.domain.name ).exists() - if ( - existing_domain_information_in_to_create is not None - or existing_domain_info - ): + if existing_domain_information_in_to_create is not None or existing_domain_info: debug_string = f"""{TerminalColors.YELLOW} Duplicate Detected: {existing_domain_information_in_to_create}. Cannot add duplicate Domain Information object @@ -601,15 +567,11 @@ class Command(BaseCommand): else: # ---------------- CREATED ---------------- domain_information_to_create.append(target_domain_information) - debug_string = ( - f"created domain information: {target_domain_information}" - ) + debug_string = f"created domain information: {target_domain_information}" elif not was_created: # ---------------- UPDATED ---------------- updated_domain_information.append(target_domain_information) - debug_string = ( - f"updated domain information: {target_domain_information}" - ) + debug_string = f"updated domain information: {target_domain_information}" else: debug_string = "domain information already exists and " f"matches incoming data (NO CHANGES MADE): {target_domain_information}" @@ -664,9 +626,7 @@ class Command(BaseCommand): # ====================================================== # ====================== DOMAIN ======================= - target_domain, was_created = self.update_or_create_domain( - transition_domain, debug_on - ) + target_domain, was_created = self.update_or_create_domain(transition_domain, debug_on) debug_string = "" if target_domain is None: @@ -704,9 +664,7 @@ class Command(BaseCommand): # ====================================================== # ================ DOMAIN INVITATIONS ================== - new_domain_invitation = self.try_add_domain_invitation( - transition_domain_email, target_domain - ) + new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, target_domain) if new_domain_invitation is None: logger.info( f"{TerminalColors.YELLOW} ! No new e-mail detected !" # noqa @@ -812,19 +770,11 @@ class Command(BaseCommand): invitation.domain = existing_domain.get() else: # Raise an err for now - raise Exception( - f"Domain {existing_domain} wants to be added" - "but doesn't exist in the DB" - ) + raise Exception(f"Domain {existing_domain} wants to be added" "but doesn't exist in the DB") invitation.save() - valid_org_choices = [ - (name, value) - for name, value in DomainApplication.OrganizationChoices.choices - ] - valid_fed_choices = [ - value for name, value in DomainApplication.BranchChoices.choices - ] + valid_org_choices = [(name, value) for name, value in DomainApplication.OrganizationChoices.choices] + valid_fed_choices = [value for name, value in DomainApplication.BranchChoices.choices] valid_agency_choices = DomainApplication.AGENCIES # ====================================================== # ================= DOMAIN INFORMATION ================= @@ -852,11 +802,7 @@ class Command(BaseCommand): TerminalHelper.print_conditional( debug_on, - ( - f"{TerminalColors.YELLOW}" - f"Trying to add: {domain_information_to_create}" - f"{TerminalColors.ENDC}" - ), + (f"{TerminalColors.YELLOW}" f"Trying to add: {domain_information_to_create}" f"{TerminalColors.ENDC}"), ) DomainInformation.objects.bulk_create(domain_information_to_create) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 488718d0b..4c431a592 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -111,9 +111,7 @@ class FileTransitionLog: """Logs every LogItem contained in this object""" for parent_log in self.logs: for child_log in parent_log: - TerminalHelper.print_conditional( - True, child_log.message, child_log.severity - ) + TerminalHelper.print_conditional(True, child_log.message, child_log.severity) def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT): """Displays all logs of a given domain_name. @@ -130,9 +128,7 @@ class FileTransitionLog: return None for log in domain_logs: - TerminalHelper.print_conditional( - restrict_type != log.code, log.message, log.code - ) + TerminalHelper.print_conditional(restrict_type != log.code, log.message, log.code) def get_logs(self, file_type, domain_name): """Grabs the logs associated with @@ -166,33 +162,21 @@ class LoadExtraTransitionDomain: updated_transition_domain = transition_domain try: # STEP 1: Parse organization data - updated_transition_domain = self.parse_org_data( - domain_name, transition_domain - ) + updated_transition_domain = self.parse_org_data(domain_name, transition_domain) # STEP 2: Parse domain type data - updated_transition_domain = self.parse_domain_type_data( - domain_name, transition_domain - ) + updated_transition_domain = self.parse_domain_type_data(domain_name, transition_domain) # STEP 3: Parse agency data - updated_transition_domain = self.parse_agency_data( - domain_name, transition_domain - ) + updated_transition_domain = self.parse_agency_data(domain_name, transition_domain) # STEP 4: Parse creation and expiration data - updated_transition_domain = self.parse_creation_expiration_data( - domain_name, transition_domain - ) + updated_transition_domain = self.parse_creation_expiration_data(domain_name, transition_domain) # Check if the instance has changed before saving updated_transition_domain.save() updated_transition_domains.append(updated_transition_domain) - logger.info( - f"{TerminalColors.OKCYAN}" - f"Successfully updated {domain_name}" - f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.OKCYAN}" f"Successfully updated {domain_name}" f"{TerminalColors.ENDC}") # If we run into an exception on this domain, # Just skip over it and log that it happened. @@ -267,8 +251,7 @@ class LoadExtraTransitionDomain: self.parse_logs.create_log_item( EnumFilenames.DOMAIN_ESCROW, LogCode.ERROR, - "Could not add epp_creation_date and epp_expiration_date " - f"on {domain_name}, no data exists.", + "Could not add epp_creation_date and epp_expiration_date " f"on {domain_name}, no data exists.", domain_name, not self.debug, ) @@ -316,10 +299,7 @@ class LoadExtraTransitionDomain: ) return transition_domain - agency_exists = ( - transition_domain.federal_agency is not None - and transition_domain.federal_agency.strip() != "" - ) + agency_exists = transition_domain.federal_agency is not None and transition_domain.federal_agency.strip() != "" if not isinstance(info.active, str) or not info.active.lower() == "y": self.parse_logs.create_log_item( @@ -354,9 +334,7 @@ class LoadExtraTransitionDomain: return transition_domain - def parse_domain_type_data( - self, domain_name, transition_domain: TransitionDomain - ) -> TransitionDomain: + def parse_domain_type_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: """Grabs organization_type and federal_type from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): @@ -401,12 +379,10 @@ class LoadExtraTransitionDomain: # Are we updating data that already exists, # or are we adding new data in its place? organization_type_exists = ( - transition_domain.organization_type is not None - and transition_domain.organization_type.strip() != "" + transition_domain.organization_type is not None and transition_domain.organization_type.strip() != "" ) federal_type_exists = ( - transition_domain.federal_type is not None - and transition_domain.federal_type.strip() != "" + transition_domain.federal_type is not None and transition_domain.federal_type.strip() != "" ) # If we get two records, then we know it is federal. @@ -440,9 +416,7 @@ class LoadExtraTransitionDomain: return transition_domain - def parse_org_data( - self, domain_name, transition_domain: TransitionDomain - ) -> TransitionDomain: + def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: """Grabs organization_name from the parsed files and associates it with a transition_domain object, then returns that object.""" if not isinstance(transition_domain, TransitionDomain): @@ -460,8 +434,7 @@ class LoadExtraTransitionDomain: return transition_domain desired_property_exists = ( - transition_domain.organization_name is not None - and transition_domain.organization_name.strip() != "" + transition_domain.organization_name is not None and transition_domain.organization_name.strip() != "" ) transition_domain.organization_name = org_info.orgname @@ -478,9 +451,7 @@ class LoadExtraTransitionDomain: return transition_domain - def _add_or_change_message( - self, file_type, var_name, changed_value, domain_name, is_update=False - ): + def _add_or_change_message(self, file_type, var_name, changed_value, domain_name, is_update=False): """Creates a log instance when a property is successfully changed on a given TransitionDomain.""" if not is_update: @@ -868,9 +839,7 @@ class ExtraTransitionDomain: else: if not infer_filenames: raise FileNotFoundError( - f"{TerminalColors.FAIL}" - f"Could not find file {filename} for {name}" - f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}" f"Could not find file {filename} for {name}" f"{TerminalColors.ENDC}" ) # Infer filename logic # @@ -903,9 +872,7 @@ class ExtraTransitionDomain: ) continue raise FileNotFoundError( - f"{TerminalColors.FAIL}" - f"Could not find file {filename} for {name}" - f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}" f"Could not find file {filename} for {name}" f"{TerminalColors.ENDC}" ) def clear_file_data(self): @@ -913,17 +880,13 @@ class ExtraTransitionDomain: file_type: FileDataHolder = item file_type.data = {} - def parse_csv_file( - self, file, seperator, dataclass_type, id_field, is_domain_escrow=False - ): + def parse_csv_file(self, file, seperator, dataclass_type, id_field, is_domain_escrow=False): # Domain escrow is an edge case if is_domain_escrow: item_to_return = self._read_domain_escrow(file, seperator) return item_to_return else: - item_to_return = self._read_csv_file( - file, seperator, dataclass_type, id_field - ) + item_to_return = self._read_csv_file(file, seperator, dataclass_type, id_field) return item_to_return # Domain escrow is an edgecase given that its structured differently data-wise. @@ -938,9 +901,7 @@ class ExtraTransitionDomain: creation_date = datetime.strptime(row[7], date_format) expiration_date = datetime.strptime(row[11], date_format) - dict_data[domain_name] = DomainEscrow( - domain_name, creation_date, expiration_date - ) + dict_data[domain_name] = DomainEscrow(domain_name, creation_date, expiration_date) return dict_data def _grab_row_id(self, row, id_field, file, dataclass_type): @@ -973,9 +934,7 @@ class ExtraTransitionDomain: f"Found bad data in {file}. Attempting to clean." f"{TerminalColors.ENDC}" ) - updated_file_content = self.replace_bad_seperators( - file, f"{seperator}", ";badseperator;" - ) + updated_file_content = self.replace_bad_seperators(file, f"{seperator}", ";badseperator;") dict_data = {} break @@ -989,11 +948,7 @@ class ExtraTransitionDomain: # After we clean the data, try to parse it again if updated_file_content: - logger.info( - f"{TerminalColors.MAGENTA}" - f"Retrying load for {file}" - f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.MAGENTA}" f"Retrying load for {file}" f"{TerminalColors.ENDC}") # Store the file locally rather than writing to the file. # This is to avoid potential data corruption. updated_file = io.StringIO(updated_file_content) @@ -1004,9 +959,7 @@ class ExtraTransitionDomain: # is wrong with the file. if None in row: logger.error( - f"{TerminalColors.FAIL}" - f"Corrupt data found for {row_id}. Skipping." - f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}" f"Corrupt data found for {row_id}. Skipping." f"{TerminalColors.ENDC}" ) continue diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 43e5c4145..85bfc8193 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -193,9 +193,7 @@ class TerminalHelper: return total_line @staticmethod - def print_to_file_conditional( - print_condition: bool, filename: str, file_directory: str, file_contents: str - ): + def print_to_file_conditional(print_condition: bool, filename: str, file_directory: str, file_contents: str): """Sometimes logger outputs get insanely huge.""" if print_condition: # Add a slash if the last character isn't one @@ -204,10 +202,6 @@ class TerminalHelper: # Assemble filepath filepath = f"{file_directory}{filename}.txt" # Write to file - logger.info( - f"{TerminalColors.MAGENTA}Writing to file " - f" {filepath}..." - f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.MAGENTA}Writing to file " f" {filepath}..." f"{TerminalColors.ENDC}") with open(f"{filepath}", "w+") as f: f.write(file_contents) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 3a31fb8e7..56425a7b7 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -37,26 +37,14 @@ class TransitionDomainArguments: # Filenames # # = Adhocs =# - agency_adhoc_filename: Optional[str] = field( - default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True - ) - domain_adhoc_filename: Optional[str] = field( - default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True - ) - organization_adhoc_filename: Optional[str] = field( - default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True - ) - authority_adhoc_filename: Optional[str] = field( - default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True - ) + agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True) + domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True) + organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True) + authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) # = Data files =# - domain_escrow_filename: Optional[str] = field( - default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True - ) - domain_additional_filename: Optional[str] = field( - default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True - ) + domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True) + domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) domain_contacts_filename: Optional[str] = field(default=None, repr=True) domain_statuses_filename: Optional[str] = field(default=None, repr=True) contacts_filename: Optional[str] = field(default=None, repr=True) diff --git a/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py b/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py index 742ab4453..1cbcf90c5 100644 --- a/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py +++ b/src/registrar/migrations/0045_transitiondomain_federal_agency_and_more.py @@ -27,16 +27,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name="transitiondomain", name="organization_type", - field=models.TextField( - blank=True, help_text="Type of organization", max_length=255, null=True - ), + field=models.TextField(blank=True, help_text="Type of organization", max_length=255, null=True), ), migrations.AddField( model_name="transitiondomain", name="organization_name", - field=models.TextField( - blank=True, db_index=True, help_text="Organization name", null=True - ), + field=models.TextField(blank=True, db_index=True, help_text="Organization name", null=True), ), migrations.AddField( model_name="transitiondomain", diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index a72fe3a21..86b8a0f7a 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -166,10 +166,7 @@ class DomainApplication(TimeStampedModel): "American Battle Monuments Commission", "AMTRAK", "Appalachian Regional Commission", - ( - "Appraisal Subcommittee of the Federal Financial " - "Institutions Examination Council" - ), + ("Appraisal Subcommittee of the Federal Financial " "Institutions Examination Council"), "Appraisal Subcommittee", "Architect of the Capitol", "Armed Forces Retirement Home", diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 7c4d2afe2..7751e7a9f 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -67,15 +67,11 @@ class TransitionDomain(TimeStampedModel): ) epp_creation_date = models.DateField( null=True, - help_text=( - "Duplication of registry's creation " "date saved for ease of reporting" - ), + help_text=("Duplication of registry's creation " "date saved for ease of reporting"), ) epp_expiration_date = models.DateField( null=True, - help_text=( - "Duplication of registry's expiration " "date saved for ease of reporting" - ), + help_text=("Duplication of registry's expiration " "date saved for ease of reporting"), ) def __str__(self): diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 745340b56..ef1c63acf 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -103,9 +103,7 @@ class TestMigrations(TestCase): # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter( - domain__name=transition_domain_name - ) + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) # Check Domain Invitation table matching_domain_invitations = DomainInvitation.objects.filter( email=transition_domain_email.lower(), @@ -146,12 +144,8 @@ class TestMigrations(TestCase): ) self.assertEqual(total_missing_domains, expected_missing_domains) self.assertEqual(total_duplicate_domains, expected_duplicate_domains) - self.assertEqual( - total_missing_domain_informations, expected_missing_domain_informations - ) - self.assertEqual( - total_missing_domain_invitations, expected_missing_domain_invitations - ) + self.assertEqual(total_missing_domain_informations, expected_missing_domain_informations) + self.assertEqual(total_missing_domain_invitations, expected_missing_domain_invitations) self.assertEqual(total_transition_domains, expected_total_transition_domains) self.assertEqual(total_domains, expected_total_domains) @@ -352,9 +346,7 @@ class TestMigrations(TestCase): # Simluate Logins for invite in DomainInvitation.objects.all(): # get a user with this email address - user, user_created = User.objects.get_or_create( - email=invite.email, username=invite.email - ) + user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) user.on_each_login() # Analyze the tables From 34d1b8f40926c98726b637d2273a3c546dd14c4c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:35:51 -0700 Subject: [PATCH 88/88] Update data_migration.md --- docs/operations/data_migration.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 38411e372..7290349ad 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -274,8 +274,6 @@ Directs the script to load only the first 100 entries into the table. You can a This will delete all the data in transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. ###### (arguments that override filepaths and directories if needed) -`--infer_filenames` -Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.. `--directory` Defines the directory where all data files and the JSON are stored. @@ -307,7 +305,8 @@ Defines the filename for domain type adhocs. `--authority_adhoc_filename` Defines the filename for domain type adhocs. - +`--infer_filenames` +Determines if we should infer filenames or not. This setting is not available for use in environments with the flag `settings.DEBUG` set to false, as it is intended for local development only. ### STEP 2: Transfer Transition Domain data into main Domain tables