From 7d98e15f3e25805811273fa0de04466a99731b78 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 2 Oct 2023 11:11:02 -0600 Subject: [PATCH 01/13] added script for transfering transition domains to domains Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 src/registrar/management/commands/transfer_transition_domains_to_domains.py diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py new file mode 100644 index 000000000..98b6ab24b --- /dev/null +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -0,0 +1,189 @@ +"""Load domain invitations for existing domains and their contacts.""" + +import csv +import logging +import argparse + +from collections import defaultdict + +from django.core.management import BaseCommand + +from registrar.models import TransitionDomain +from registrar.models import Domain + +logger = logging.getLogger(__name__) + + +class termColors: + """Colors for terminal outputs + (makes reading the logs WAY easier)""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + BackgroundLightYellow = "\033[103m" + +#---------------------------------------- +# MAIN SCRIPT +#---------------------------------------- +class Command(BaseCommand): + help = """Load data from transition domain tables + into main domain tables.""" + + def add_arguments(self, parser): + + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", default=0, help="Sets max number of entries to load" + ) + + + def handle( # noqa: C901 + self, + **options, + ): + """Load the data files and create the DomainInvitations.""" + sep = options.get("sep") + + # grab command line arguments and store locally... + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + + self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + + # domains to ADD + to_create = [] + # domains we UPDATED + updated_domain_entries = [] + # if we are limiting our parse (for testing purposes, keep + # track of total rows parsed) + total_rows_parsed = 0 + + logger.info(f"""{termColors.OKGREEN} +========================== +Beginning Data Transfer +========================== +{termColors.ENDC}""") + + + for transition_entry in TransitionDomain.objects.all(): + transition_domain_name = transition_entry.domain_name + transition_domain_status = transition_entry.status + + # Check for existing domain entry + try: + # for existing entry, update the status to the transition domain status + existingEntry = Domain.objects.get( + name=transition_domain_name + ) + if transition_domain_status is TransitionDomain.StatusChoices.HOLD: + existingEntry.place_client_hold() + else: + existingEntry._remove_client_hold() + updated_domain_entries.append(existingEntry) + + # DEBUG: + if debug_on: + logger.info(f"""{termColors.WARNING} + Updated Domain: {existingEntry}""") + except Domain.DoesNotExist: + # no matching entry, make one + newEntry = Domain( + name = transition_domain_name, + state = transition_domain_status + ) + to_create.append(newEntry) + + # DEBUG: + if debug_on: + logger.info( + f"{termColors.OKCYAN} Adding entry: {newEntry} {termColors.ENDC}" # noqa + ) + except Domain.MultipleObjectsReturned: + logger.info( + f""" +{termColors.FAIL} +!!! ERROR: duplicate entries exist in the +Domain table for domain: +{transition_domain_name} +----------TERMINATING----------""" + ) + import sys + sys.exit() + + # DEBUG: + if debug_on or debug_max_entries_to_parse > 0: + if ( + total_rows_parsed > debug_max_entries_to_parse + and debug_max_entries_to_parse != 0 + ): + logger.info( + f"""{termColors.WARNING} + ----PARSE LIMIT REACHED. HALTING PARSER.---- + {termColors.ENDC} + """ + ) + break + + Domain.objects.bulk_create(to_create) + + total_new_entries = len(to_create) + total_updated_domain_entries = len(updated_domain_entries) + + logger.info( +f"""{termColors.OKGREEN} + +============= FINISHED =============== +Created {total_new_entries} transition domain entries, +updated {total_updated_domain_entries} transition domain entries +{termColors.ENDC} +""" +) + + # DEBUG: + if debug_on: + logger.info( +f"""{termColors.WARNING} + +Created Domains: +{to_create} + +Updated Domains: +{updated_domain_entries} + +{termColors.ENDC} +""" +) + + +# ---------------------------------------- +# HELPER FUNCTIONS +# ---------------------------------------- + + def print_debug_mode_statements(self, debug_on, debug_max_entries_to_parse): + if debug_on: + logger.info( + f"""{termColors.WARNING} +----------DEBUG MODE ON---------- +Detailed print statements activated. +{termColors.ENDC} +""" +) + if debug_max_entries_to_parse > 0: + logger.info( + f"""{termColors.OKCYAN} +----------LIMITER ON---------- +Data transfer will be limited to +{debug_max_entries_to_parse} entries.") +{termColors.ENDC} +""" +) \ No newline at end of file From 1633feb57bfe50ba189fd239762a4892e0a8fab6 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 2 Oct 2023 23:43:22 -0600 Subject: [PATCH 02/13] -s --- .../transfer_transition_domains_to_domains.py | 47 ++++++++++++++++--- .../0033_alter_transitiondomain_status.py | 24 ++++++++++ src/registrar/models/domain.py | 19 +++++--- src/registrar/models/transition_domain.py | 6 ++- 4 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 src/registrar/migrations/0033_alter_transitiondomain_status.py diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 98b6ab24b..db762d84a 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -5,6 +5,7 @@ import logging import argparse from collections import defaultdict +from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand @@ -64,6 +65,8 @@ class Command(BaseCommand): to_create = [] # domains we UPDATED updated_domain_entries = [] + # domains we SKIPPED + skipped_domain_entries = [] # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 @@ -81,20 +84,32 @@ Beginning Data Transfer # Check for existing domain entry try: + # DEBUG: + if debug_on: + logger.info(f"""{termColors.WARNING} +Processing Transition Domain: {transition_domain_name}, {transition_domain_status}{termColors.ENDC}""") + # for existing entry, update the status to the transition domain status existingEntry = Domain.objects.get( name=transition_domain_name ) - if transition_domain_status is TransitionDomain.StatusChoices.HOLD: - existingEntry.place_client_hold() - else: - existingEntry._remove_client_hold() - updated_domain_entries.append(existingEntry) - + current_state = existingEntry.state + # DEBUG: if debug_on: logger.info(f"""{termColors.WARNING} - Updated Domain: {existingEntry}""") + > Found existing domain entry for: {transition_domain_name}, {current_state}{termColors.ENDC}""") + if transition_domain_status != current_state: + if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: + existingEntry.place_client_hold(ignoreEPP=True) + else: + existingEntry.revert_client_hold(ignoreEPP=True) + existingEntry.save() + updated_domain_entries.append(existingEntry) + # DEBUG: + if debug_on: + logger.info(f"""{termColors.WARNING} + >> Updated {transition_domain_name} state from '{current_state}' to '{existingEntry.state}{termColors.ENDC}'""") except Domain.DoesNotExist: # no matching entry, make one newEntry = Domain( @@ -119,6 +134,15 @@ Domain table for domain: ) import sys sys.exit() + except TransitionNotAllowed as err: + skipped_domain_entries.append(transition_domain_name) + logger.info( + f"""{termColors.FAIL} +Unable to change state for {transition_domain_name} + TRANSITION NOT ALLOWED error message (internal): + {err} + ----------SKIPPING----------""" + ) # DEBUG: if debug_on or debug_max_entries_to_parse > 0: @@ -147,6 +171,15 @@ Created {total_new_entries} transition domain entries, updated {total_updated_domain_entries} transition domain entries {termColors.ENDC} """ +) + if len(skipped_domain_entries) > 0: + logger.info( +f"""{termColors.FAIL} + +============= SKIPPED DOMAINS (ERRORS) =============== +{skipped_domain_entries} +{termColors.ENDC} +""" ) # DEBUG: diff --git a/src/registrar/migrations/0033_alter_transitiondomain_status.py b/src/registrar/migrations/0033_alter_transitiondomain_status.py new file mode 100644 index 000000000..7e9ca6ffa --- /dev/null +++ b/src/registrar/migrations/0033_alter_transitiondomain_status.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.1 on 2023-10-03 05:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0032_alter_transitiondomain_status"), + ] + + operations = [ + migrations.AlterField( + model_name="transitiondomain", + name="status", + field=models.CharField( + blank=True, + choices=[("ready", "Ready"), ("on hold", "On Hold")], + default="ready", + help_text="domain status during the transfer", + max_length=255, + verbose_name="Status", + ), + ), + ] diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index e45724a9b..da3f4c06a 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -783,20 +783,27 @@ class Domain(TimeStampedModel, DomainHelper): @transition( field="state", source=[State.READY, State.ON_HOLD], target=State.ON_HOLD ) - def place_client_hold(self): - """place a clienthold on a domain (no longer should resolve)""" + def place_client_hold(self, ignoreEPP=False): + """place a clienthold on a domain (no longer should resolve) + ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains)""" # TODO - ensure all requirements for client hold are made here # (check prohibited statuses) logger.info("clientHold()-> inside clientHold") - self._place_client_hold() + + # In order to allow transition domains to by-pass EPP calls, + # include this ignoreEPP flag + if not ignoreEPP: + self._place_client_hold() # TODO -on the client hold ticket any additional error handling here @transition(field="state", source=[State.READY, State.ON_HOLD], target=State.READY) - def revert_client_hold(self): - """undo a clienthold placed on a domain""" + def revert_client_hold(self, ignoreEPP=False): + """undo a clienthold placed on a domain + ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains)""" logger.info("clientHold()-> inside clientHold") - self._remove_client_hold() + if not ignoreEPP: + self._remove_client_hold() # TODO -on the client hold ticket any additional error handling here @transition( diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 203795925..fa796dd18 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -5,7 +5,7 @@ from .utility.time_stamped_model import TimeStampedModel class StatusChoices(models.TextChoices): READY = "ready", "Ready" - HOLD = "hold", "Hold" + ON_HOLD = "on hold", "On Hold" class TransitionDomain(TimeStampedModel): @@ -13,6 +13,10 @@ class TransitionDomain(TimeStampedModel): state of a domain upon transition between registry providers""" + # This is necessary to expose the enum to external + # classes that import TransitionDomain + StatusChoices = StatusChoices + username = models.TextField( null=False, blank=False, From 688b5742c99bd0a1285c8eca49c55bf1a06535ed Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 3 Oct 2023 00:13:42 -0600 Subject: [PATCH 03/13] linted Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 91 ++++++++++--------- src/registrar/models/domain.py | 6 +- src/registrar/models/transition_domain.py | 2 +- 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index db762d84a..016d1c943 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -1,11 +1,9 @@ """Load domain invitations for existing domains and their contacts.""" -import csv import logging import argparse -from collections import defaultdict -from django_fsm import TransitionNotAllowed # type: ignore +from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand @@ -30,28 +28,26 @@ class termColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" -#---------------------------------------- + +# ---------------------------------------- # MAIN SCRIPT -#---------------------------------------- +# ---------------------------------------- class Command(BaseCommand): - help = """Load data from transition domain tables + help = """Load data from transition domain tables into main domain tables.""" def add_arguments(self, parser): - parser.add_argument("--debug", action=argparse.BooleanOptionalAction) parser.add_argument( "--limitParse", default=0, help="Sets max number of entries to load" ) - def handle( # noqa: C901 self, **options, ): """Load the data files and create the DomainInvitations.""" - sep = options.get("sep") # grab command line arguments and store locally... debug_on = options.get("debug") @@ -71,12 +67,13 @@ class Command(BaseCommand): # track of total rows parsed) total_rows_parsed = 0 - logger.info(f"""{termColors.OKGREEN} -========================== -Beginning Data Transfer + logger.info( + f"""{termColors.OKGREEN} ========================== -{termColors.ENDC}""") - +Beginning Data Transfer +========================== +{termColors.ENDC}""" + ) for transition_entry in TransitionDomain.objects.all(): transition_domain_name = transition_entry.domain_name @@ -86,21 +83,29 @@ Beginning Data Transfer try: # DEBUG: if debug_on: - logger.info(f"""{termColors.WARNING} -Processing Transition Domain: {transition_domain_name}, {transition_domain_status}{termColors.ENDC}""") + logger.info( + f"""{termColors.WARNING} +Processing Transition Domain: {transition_domain_name}, {transition_domain_status} +{termColors.ENDC}""" + ) - # for existing entry, update the status to the transition domain status - existingEntry = Domain.objects.get( - name=transition_domain_name - ) + # for existing entry, update the status to + # the transition domain status + existingEntry = Domain.objects.get(name=transition_domain_name) current_state = existingEntry.state - + # DEBUG: if debug_on: - logger.info(f"""{termColors.WARNING} - > Found existing domain entry for: {transition_domain_name}, {current_state}{termColors.ENDC}""") + logger.info( + f"""{termColors.WARNING} + > Found existing domain entry for: {transition_domain_name}, {current_state} + {termColors.ENDC}""" + ) if transition_domain_status != current_state: - if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: + if ( + transition_domain_status + == TransitionDomain.StatusChoices.ON_HOLD + ): existingEntry.place_client_hold(ignoreEPP=True) else: existingEntry.revert_client_hold(ignoreEPP=True) @@ -108,13 +113,15 @@ Processing Transition Domain: {transition_domain_name}, {transition_domain_statu updated_domain_entries.append(existingEntry) # DEBUG: if debug_on: - logger.info(f"""{termColors.WARNING} - >> Updated {transition_domain_name} state from '{current_state}' to '{existingEntry.state}{termColors.ENDC}'""") + logger.info( + f"""{termColors.WARNING} + >> Updated {transition_domain_name} state from + '{current_state}' to '{existingEntry.state}{termColors.ENDC}'""" + ) except Domain.DoesNotExist: # no matching entry, make one newEntry = Domain( - name = transition_domain_name, - state = transition_domain_status + name=transition_domain_name, state=transition_domain_status ) to_create.append(newEntry) @@ -131,8 +138,9 @@ Processing Transition Domain: {transition_domain_name}, {transition_domain_statu Domain table for domain: {transition_domain_name} ----------TERMINATING----------""" - ) + ) import sys + sys.exit() except TransitionNotAllowed as err: skipped_domain_entries.append(transition_domain_name) @@ -142,7 +150,7 @@ Unable to change state for {transition_domain_name} TRANSITION NOT ALLOWED error message (internal): {err} ----------SKIPPING----------""" - ) + ) # DEBUG: if debug_on or debug_max_entries_to_parse > 0: @@ -164,28 +172,28 @@ Unable to change state for {transition_domain_name} total_updated_domain_entries = len(updated_domain_entries) logger.info( -f"""{termColors.OKGREEN} + f"""{termColors.OKGREEN} ============= FINISHED =============== Created {total_new_entries} transition domain entries, updated {total_updated_domain_entries} transition domain entries {termColors.ENDC} """ -) + ) if len(skipped_domain_entries) > 0: logger.info( -f"""{termColors.FAIL} + f"""{termColors.FAIL} ============= SKIPPED DOMAINS (ERRORS) =============== {skipped_domain_entries} {termColors.ENDC} """ -) + ) # DEBUG: if debug_on: logger.info( -f"""{termColors.WARNING} + f"""{termColors.WARNING} Created Domains: {to_create} @@ -195,12 +203,11 @@ Updated Domains: {termColors.ENDC} """ -) + ) - -# ---------------------------------------- -# HELPER FUNCTIONS -# ---------------------------------------- + # ---------------------------------------- + # HELPER FUNCTIONS + # ---------------------------------------- def print_debug_mode_statements(self, debug_on, debug_max_entries_to_parse): if debug_on: @@ -210,7 +217,7 @@ Updated Domains: Detailed print statements activated. {termColors.ENDC} """ -) + ) if debug_max_entries_to_parse > 0: logger.info( f"""{termColors.OKCYAN} @@ -219,4 +226,4 @@ Data transfer will be limited to {debug_max_entries_to_parse} entries.") {termColors.ENDC} """ -) \ No newline at end of file + ) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index da3f4c06a..a5076a32e 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -785,7 +785,8 @@ class Domain(TimeStampedModel, DomainHelper): ) def place_client_hold(self, ignoreEPP=False): """place a clienthold on a domain (no longer should resolve) - ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains)""" + ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains) + """ # TODO - ensure all requirements for client hold are made here # (check prohibited statuses) logger.info("clientHold()-> inside clientHold") @@ -799,7 +800,8 @@ class Domain(TimeStampedModel, DomainHelper): @transition(field="state", source=[State.READY, State.ON_HOLD], target=State.READY) def revert_client_hold(self, ignoreEPP=False): """undo a clienthold placed on a domain - ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains)""" + ignoreEPP (boolean) - set to true to by-pass EPP (used for transition domains) + """ logger.info("clientHold()-> inside clientHold") if not ignoreEPP: diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index fa796dd18..232fd9033 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -13,7 +13,7 @@ class TransitionDomain(TimeStampedModel): state of a domain upon transition between registry providers""" - # This is necessary to expose the enum to external + # This is necessary to expose the enum to external # classes that import TransitionDomain StatusChoices = StatusChoices From 604821ad07cb3feed9f0ba5f9a39fb4af34e645d Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 6 Oct 2023 03:19:09 -0600 Subject: [PATCH 04/13] added domain invitation entries Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 170 ++++++++++-------- 1 file changed, 96 insertions(+), 74 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 016d1c943..3cdef793b 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -9,6 +9,7 @@ from django.core.management import BaseCommand from registrar.models import TransitionDomain from registrar.models import Domain +from registrar.models import DomainInvitation logger = logging.getLogger(__name__) @@ -21,7 +22,7 @@ class termColors: OKBLUE = "\033[94m" OKCYAN = "\033[96m" OKGREEN = "\033[92m" - WARNING = "\033[93m" + YELLOW = "\033[93m" FAIL = "\033[91m" ENDC = "\033[0m" BOLD = "\033[1m" @@ -29,25 +30,53 @@ class termColors: BackgroundLightYellow = "\033[103m" -# ---------------------------------------- -# MAIN SCRIPT -# ---------------------------------------- + def print_debug_mode_statements(self, + debug_on: bool, + debug_max_entries_to_parse: int): + """Prints additional terminal statements to indicate if --debug + or --limitParse are in use""" + if debug_on: + logger.info( + f"""{termColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {termColors.ENDC} + """ + ) + if debug_max_entries_to_parse > 0: + logger.info( + f"""{termColors.OKCYAN} + ----------LIMITER ON---------- + Parsing of entries will be limited to + {debug_max_entries_to_parse} lines per file.") + Detailed print statements activated. + {termColors.ENDC} + """ + ) + + class Command(BaseCommand): help = """Load data from transition domain tables - into main domain tables.""" + into main domain tables. Also create domain invitation + entries for every domain we ADD (but not for domains + we UPDATE)""" def add_arguments(self, parser): parser.add_argument("--debug", action=argparse.BooleanOptionalAction) parser.add_argument( - "--limitParse", default=0, help="Sets max number of entries to load" + "--limitParse", + default=0, + help="Sets max number of entries to load, set to 0 to load all entries" ) def handle( # noqa: C901 self, **options, ): - """Load the data files and create the DomainInvitations.""" + """Parse entries in TransitionDomain table + and create (or update) corresponding entries in the + Domain and DomainInvitation tables.""" # grab command line arguments and store locally... debug_on = options.get("debug") @@ -58,7 +87,8 @@ class Command(BaseCommand): self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) # domains to ADD - to_create = [] + domains_to_create = [] + domain_invitations_to_create = [] # domains we UPDATED updated_domain_entries = [] # domains we SKIPPED @@ -69,24 +99,25 @@ class Command(BaseCommand): logger.info( f"""{termColors.OKGREEN} -========================== -Beginning Data Transfer -========================== -{termColors.ENDC}""" + ========================== + Beginning Data Transfer + ========================== + {termColors.ENDC}""" ) for transition_entry in TransitionDomain.objects.all(): transition_domain_name = transition_entry.domain_name transition_domain_status = transition_entry.status + transition_domain_email = transition_entry.username # Check for existing domain entry try: # DEBUG: if debug_on: logger.info( - f"""{termColors.WARNING} -Processing Transition Domain: {transition_domain_name}, {transition_domain_status} -{termColors.ENDC}""" + f"""{termColors.YELLOW} + Processing Transition Domain: {transition_domain_name}, {transition_domain_status} + {termColors.ENDC}""" # noqa ) # for existing entry, update the status to @@ -97,9 +128,9 @@ Processing Transition Domain: {transition_domain_name}, {transition_domain_statu # DEBUG: if debug_on: logger.info( - f"""{termColors.WARNING} - > Found existing domain entry for: {transition_domain_name}, {current_state} - {termColors.ENDC}""" + f"""{termColors.YELLOW} + > Found existing domain entry for: {transition_domain_name}, {current_state} + {termColors.ENDC}""" # noqa ) if transition_domain_status != current_state: if ( @@ -114,30 +145,39 @@ Processing Transition Domain: {transition_domain_name}, {transition_domain_statu # DEBUG: if debug_on: logger.info( - f"""{termColors.WARNING} - >> Updated {transition_domain_name} state from - '{current_state}' to '{existingEntry.state}{termColors.ENDC}'""" + f"""{termColors.YELLOW} + >> Updated {transition_domain_name} state from + '{current_state}' to '{existingEntry.state}{termColors.ENDC}' + (no domain invitation entry added)""" ) except Domain.DoesNotExist: # no matching entry, make one newEntry = Domain( - name=transition_domain_name, state=transition_domain_status + name=transition_domain_name, + state=transition_domain_status + ) + domains_to_create.append(newEntry) + + domain_invitations_to_create.append( + DomainInvitation( + email=transition_domain_email.lower(), + domain=transition_domain_name + ) ) - to_create.append(newEntry) # DEBUG: if debug_on: logger.info( - f"{termColors.OKCYAN} Adding entry: {newEntry} {termColors.ENDC}" # noqa + f"{termColors.OKCYAN} Adding domain AND domain invitation: {newEntry} {termColors.ENDC}" # noqa ) except Domain.MultipleObjectsReturned: logger.info( f""" -{termColors.FAIL} -!!! ERROR: duplicate entries exist in the -Domain table for domain: -{transition_domain_name} -----------TERMINATING----------""" + {termColors.FAIL} + !!! ERROR: duplicate entries exist in the + Domain table for domain: + {transition_domain_name} + ----------TERMINATING----------""" ) import sys @@ -146,10 +186,10 @@ Domain table for domain: skipped_domain_entries.append(transition_domain_name) logger.info( f"""{termColors.FAIL} -Unable to change state for {transition_domain_name} - TRANSITION NOT ALLOWED error message (internal): - {err} - ----------SKIPPING----------""" + Unable to change state for {transition_domain_name} + TRANSITION NOT ALLOWED error message (internal): + {err} + ----------SKIPPING----------""" ) # DEBUG: @@ -159,71 +199,53 @@ Unable to change state for {transition_domain_name} and debug_max_entries_to_parse != 0 ): logger.info( - f"""{termColors.WARNING} + f"""{termColors.YELLOW} ----PARSE LIMIT REACHED. HALTING PARSER.---- {termColors.ENDC} """ ) break - Domain.objects.bulk_create(to_create) + Domain.objects.bulk_create(domains_to_create) + DomainInvitation.objects.bulk_create(domain_invitations_to_create) - total_new_entries = len(to_create) + total_new_entries = len(domains_to_create) total_updated_domain_entries = len(updated_domain_entries) + total_domain_invitation_entries = len(domain_invitations_to_create) logger.info( f"""{termColors.OKGREEN} + ============= FINISHED =============== + Created {total_new_entries} transition domain entries, + updated {total_updated_domain_entries} transition domain entries -============= FINISHED =============== -Created {total_new_entries} transition domain entries, -updated {total_updated_domain_entries} transition domain entries -{termColors.ENDC} -""" + Created {total_domain_invitation_entries} domain invitation entries + (NOTE: no invitations are SENT in this script) + {termColors.ENDC} + """ ) if len(skipped_domain_entries) > 0: logger.info( f"""{termColors.FAIL} - -============= SKIPPED DOMAINS (ERRORS) =============== -{skipped_domain_entries} -{termColors.ENDC} -""" + ============= SKIPPED DOMAINS (ERRORS) =============== + {skipped_domain_entries} + {termColors.ENDC} + """ ) # DEBUG: if debug_on: logger.info( - f"""{termColors.WARNING} + f"""{termColors.YELLOW} -Created Domains: -{to_create} + Created Domains: + {domains_to_create} -Updated Domains: -{updated_domain_entries} + Updated Domains: + {updated_domain_entries} -{termColors.ENDC} -""" + {termColors.ENDC} + """ ) - # ---------------------------------------- - # HELPER FUNCTIONS - # ---------------------------------------- - def print_debug_mode_statements(self, debug_on, debug_max_entries_to_parse): - if debug_on: - logger.info( - f"""{termColors.WARNING} -----------DEBUG MODE ON---------- -Detailed print statements activated. -{termColors.ENDC} -""" - ) - if debug_max_entries_to_parse > 0: - logger.info( - f"""{termColors.OKCYAN} -----------LIMITER ON---------- -Data transfer will be limited to -{debug_max_entries_to_parse} entries.") -{termColors.ENDC} -""" - ) From bede4c86075e98c7df82827ec1ff400a52bfa29f Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 6 Oct 2023 03:23:16 -0600 Subject: [PATCH 05/13] linted Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 3cdef793b..98d00e4b0 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -29,10 +29,9 @@ class termColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" - - def print_debug_mode_statements(self, - debug_on: bool, - debug_max_entries_to_parse: int): + def print_debug_mode_statements( + self, debug_on: bool, debug_max_entries_to_parse: int + ): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" if debug_on: @@ -65,9 +64,9 @@ class Command(BaseCommand): parser.add_argument("--debug", action=argparse.BooleanOptionalAction) parser.add_argument( - "--limitParse", - default=0, - help="Sets max number of entries to load, set to 0 to load all entries" + "--limitParse", + default=0, + help="Sets max number of entries to load, set to 0 to load all entries", ) def handle( # noqa: C901 @@ -75,7 +74,7 @@ class Command(BaseCommand): **options, ): """Parse entries in TransitionDomain table - and create (or update) corresponding entries in the + and create (or update) corresponding entries in the Domain and DomainInvitation tables.""" # grab command line arguments and store locally... @@ -117,7 +116,7 @@ class Command(BaseCommand): logger.info( f"""{termColors.YELLOW} Processing Transition Domain: {transition_domain_name}, {transition_domain_status} - {termColors.ENDC}""" # noqa + {termColors.ENDC}""" # noqa ) # for existing entry, update the status to @@ -130,7 +129,7 @@ class Command(BaseCommand): logger.info( f"""{termColors.YELLOW} > Found existing domain entry for: {transition_domain_name}, {current_state} - {termColors.ENDC}""" # noqa + {termColors.ENDC}""" # noqa ) if transition_domain_status != current_state: if ( @@ -147,21 +146,21 @@ class Command(BaseCommand): logger.info( f"""{termColors.YELLOW} >> Updated {transition_domain_name} state from - '{current_state}' to '{existingEntry.state}{termColors.ENDC}' - (no domain invitation entry added)""" + '{current_state}' to '{existingEntry.state}' + (no domain invitation entry added) + {termColors.ENDC}""" ) except Domain.DoesNotExist: # no matching entry, make one newEntry = Domain( - name=transition_domain_name, - state=transition_domain_status + name=transition_domain_name, state=transition_domain_status ) domains_to_create.append(newEntry) domain_invitations_to_create.append( DomainInvitation( email=transition_domain_email.lower(), - domain=transition_domain_name + domain=transition_domain_name, ) ) @@ -247,5 +246,3 @@ class Command(BaseCommand): {termColors.ENDC} """ ) - - From 7f7a6f1fdb67e52abea57bfe98141d39e220816d Mon Sep 17 00:00:00 2001 From: CocoByte Date: Sun, 8 Oct 2023 19:25:43 -0600 Subject: [PATCH 06/13] Domain Invitation portion updated to catch anomalies Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 109 +++++++++++------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 98d00e4b0..b1c33c3e7 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -2,6 +2,7 @@ import logging import argparse +import sys from django_fsm import TransitionNotAllowed # type: ignore @@ -29,6 +30,23 @@ class termColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" + + +class Command(BaseCommand): + help = """Load data from transition domain tables + into main domain tables. Also create domain invitation + entries for every domain we ADD (but not for domains + we UPDATE)""" + + def add_arguments(self, parser): + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", + default=0, + help="Sets max number of entries to load, set to 0 to load all entries", + ) + def print_debug_mode_statements( self, debug_on: bool, debug_max_entries_to_parse: int ): @@ -54,21 +72,6 @@ class termColors: ) -class Command(BaseCommand): - help = """Load data from transition domain tables - into main domain tables. Also create domain invitation - entries for every domain we ADD (but not for domains - we UPDATE)""" - - def add_arguments(self, parser): - parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - - parser.add_argument( - "--limitParse", - default=0, - help="Sets max number of entries to load, set to 0 to load all entries", - ) - def handle( # noqa: C901 self, **options, @@ -92,6 +95,8 @@ class Command(BaseCommand): updated_domain_entries = [] # domains we SKIPPED skipped_domain_entries = [] + # domainInvitations we SKIPPED + skipped_domain_invitations = [] # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 @@ -114,8 +119,8 @@ class Command(BaseCommand): # DEBUG: if debug_on: logger.info( - f"""{termColors.YELLOW} - Processing Transition Domain: {transition_domain_name}, {transition_domain_status} + f"""{termColors.OKCYAN} + Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} {termColors.ENDC}""" # noqa ) @@ -132,10 +137,7 @@ class Command(BaseCommand): {termColors.ENDC}""" # noqa ) if transition_domain_status != current_state: - if ( - transition_domain_status - == TransitionDomain.StatusChoices.ON_HOLD - ): + if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: existingEntry.place_client_hold(ignoreEPP=True) else: existingEntry.revert_client_hold(ignoreEPP=True) @@ -151,24 +153,46 @@ class Command(BaseCommand): {termColors.ENDC}""" ) except Domain.DoesNotExist: - # no matching entry, make one - newEntry = Domain( - name=transition_domain_name, state=transition_domain_status - ) - domains_to_create.append(newEntry) - domain_invitations_to_create.append( - DomainInvitation( - email=transition_domain_email.lower(), - domain=transition_domain_name, - ) + already_in_to_create = next( + (x for x in domains_to_create if x.name == transition_domain_name), + None, ) - - # DEBUG: - if debug_on: - logger.info( - f"{termColors.OKCYAN} Adding domain AND domain invitation: {newEntry} {termColors.ENDC}" # noqa + if already_in_to_create: + # DEBUG: + if debug_on: + logger.info( + f"""{termColors.YELLOW} + Duplicate Detected: {transition_domain_name}. + Cannot add duplicate entry for another username. + Violates Unique Key constraint. + {termColors.ENDC}""" + ) + else: + # no matching entry, make one + new_entry = Domain( + name=transition_domain_name, state=transition_domain_status ) + domains_to_create.append(new_entry) + + if transition_domain_email: + new_domain_invitation = DomainInvitation( + email=transition_domain_email.lower(), + domain=new_entry + ) + domain_invitations_to_create.append(new_domain_invitation) + else: + logger.info( + f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" + f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" + ) + skipped_domain_invitations.append(transition_domain_name) + + # DEBUG: + if debug_on: + logger.info( + f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}" # noqa + ) except Domain.MultipleObjectsReturned: logger.info( f""" @@ -178,8 +202,6 @@ class Command(BaseCommand): {transition_domain_name} ----------TERMINATING----------""" ) - import sys - sys.exit() except TransitionNotAllowed as err: skipped_domain_entries.append(transition_domain_name) @@ -194,7 +216,7 @@ class Command(BaseCommand): # DEBUG: if debug_on or debug_max_entries_to_parse > 0: if ( - total_rows_parsed > debug_max_entries_to_parse + total_rows_parsed >= debug_max_entries_to_parse and debug_max_entries_to_parse != 0 ): logger.info( @@ -231,6 +253,15 @@ class Command(BaseCommand): {termColors.ENDC} """ ) + if len(skipped_domain_invitations) > 0: + logger.info( + f"""{termColors.FAIL} + ============= SKIPPED DOMAIN INVITATIONS (ERRORS) =============== + {skipped_domain_invitations} + {termColors.ENDC} + """ + ) + # DEBUG: if debug_on: From 3ea08112392177b0edd1245c5d829fed074cb144 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 10 Oct 2023 13:56:36 -0600 Subject: [PATCH 07/13] fixed migration conflicts Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 16 +++++++--------- ....py => 0038_alter_transitiondomain_status.py} | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) rename src/registrar/migrations/{0033_alter_transitiondomain_status.py => 0038_alter_transitiondomain_status.py} (83%) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index b1c33c3e7..37f894806 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -30,7 +30,6 @@ class termColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" - class Command(BaseCommand): help = """Load data from transition domain tables @@ -71,7 +70,6 @@ class Command(BaseCommand): """ ) - def handle( # noqa: C901 self, **options, @@ -137,7 +135,10 @@ class Command(BaseCommand): {termColors.ENDC}""" # noqa ) if transition_domain_status != current_state: - if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: + if ( + transition_domain_status + == TransitionDomain.StatusChoices.ON_HOLD + ): existingEntry.place_client_hold(ignoreEPP=True) else: existingEntry.revert_client_hold(ignoreEPP=True) @@ -153,7 +154,6 @@ class Command(BaseCommand): {termColors.ENDC}""" ) except Domain.DoesNotExist: - already_in_to_create = next( (x for x in domains_to_create if x.name == transition_domain_name), None, @@ -177,13 +177,12 @@ class Command(BaseCommand): if transition_domain_email: new_domain_invitation = DomainInvitation( - email=transition_domain_email.lower(), - domain=new_entry - ) + email=transition_domain_email.lower(), domain=new_entry + ) domain_invitations_to_create.append(new_domain_invitation) else: logger.info( - f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" + f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" ) skipped_domain_invitations.append(transition_domain_name) @@ -261,7 +260,6 @@ class Command(BaseCommand): {termColors.ENDC} """ ) - # DEBUG: if debug_on: diff --git a/src/registrar/migrations/0033_alter_transitiondomain_status.py b/src/registrar/migrations/0038_alter_transitiondomain_status.py similarity index 83% rename from src/registrar/migrations/0033_alter_transitiondomain_status.py rename to src/registrar/migrations/0038_alter_transitiondomain_status.py index 7e9ca6ffa..5e83e7076 100644 --- a/src/registrar/migrations/0033_alter_transitiondomain_status.py +++ b/src/registrar/migrations/0038_alter_transitiondomain_status.py @@ -1,11 +1,11 @@ -# Generated by Django 4.2.1 on 2023-10-03 05:43 +# Generated by Django 4.2.1 on 2023-10-10 17:38 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("registrar", "0032_alter_transitiondomain_status"), + ("registrar", "0037_create_groups_v01"), ] operations = [ From b3b08d57a3e1c6a0f242d520c193828ee91bc863 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 10 Oct 2023 14:58:40 -0600 Subject: [PATCH 08/13] linted..again Signed-off-by: CocoByte --- .../commands/transfer_transition_domains_to_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 37f894806..55670c174 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -182,7 +182,7 @@ class Command(BaseCommand): domain_invitations_to_create.append(new_domain_invitation) else: logger.info( - f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" + f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" # noqa f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" ) skipped_domain_invitations.append(transition_domain_name) From b8a571811349756b0ebbe62a29f82ca22ea63365 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 11 Oct 2023 16:12:15 -0600 Subject: [PATCH 09/13] Refactored handle() to be less complex, satisfied linter Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 339 ++++++++++-------- 1 file changed, 184 insertions(+), 155 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 55670c174..0e27bf20d 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -70,164 +70,46 @@ class Command(BaseCommand): """ ) - def handle( # noqa: C901 - self, - **options, - ): - """Parse entries in TransitionDomain table - and create (or update) corresponding entries in the - Domain and DomainInvitation tables.""" - - # grab command line arguments and store locally... - debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) # set to 0 to parse all entries - - self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) - - # domains to ADD - domains_to_create = [] - domain_invitations_to_create = [] - # domains we UPDATED - updated_domain_entries = [] - # domains we SKIPPED - skipped_domain_entries = [] - # domainInvitations we SKIPPED - skipped_domain_invitations = [] - # if we are limiting our parse (for testing purposes, keep - # track of total rows parsed) - total_rows_parsed = 0 - - logger.info( - f"""{termColors.OKGREEN} - ========================== - Beginning Data Transfer - ========================== - {termColors.ENDC}""" - ) - - for transition_entry in TransitionDomain.objects.all(): - transition_domain_name = transition_entry.domain_name - transition_domain_status = transition_entry.status - transition_domain_email = transition_entry.username - - # Check for existing domain entry - try: - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.OKCYAN} - Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} - {termColors.ENDC}""" # noqa - ) - - # for existing entry, update the status to - # the transition domain status - existingEntry = Domain.objects.get(name=transition_domain_name) - current_state = existingEntry.state - - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - > Found existing domain entry for: {transition_domain_name}, {current_state} - {termColors.ENDC}""" # noqa - ) - if transition_domain_status != current_state: - if ( - transition_domain_status - == TransitionDomain.StatusChoices.ON_HOLD - ): - existingEntry.place_client_hold(ignoreEPP=True) - else: - existingEntry.revert_client_hold(ignoreEPP=True) - existingEntry.save() - updated_domain_entries.append(existingEntry) - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - >> Updated {transition_domain_name} state from - '{current_state}' to '{existingEntry.state}' - (no domain invitation entry added) - {termColors.ENDC}""" - ) - except Domain.DoesNotExist: - already_in_to_create = next( - (x for x in domains_to_create if x.name == transition_domain_name), - None, - ) - if already_in_to_create: - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - Duplicate Detected: {transition_domain_name}. - Cannot add duplicate entry for another username. - Violates Unique Key constraint. - {termColors.ENDC}""" - ) - else: - # no matching entry, make one - new_entry = Domain( - name=transition_domain_name, state=transition_domain_status - ) - domains_to_create.append(new_entry) - - if transition_domain_email: - new_domain_invitation = DomainInvitation( - email=transition_domain_email.lower(), domain=new_entry - ) - domain_invitations_to_create.append(new_domain_invitation) - else: - logger.info( - f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" # noqa - f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" - ) - skipped_domain_invitations.append(transition_domain_name) - - # DEBUG: - if debug_on: - logger.info( - f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}" # noqa - ) - except Domain.MultipleObjectsReturned: - logger.info( - f""" - {termColors.FAIL} - !!! ERROR: duplicate entries exist in the - Domain table for domain: - {transition_domain_name} - ----------TERMINATING----------""" - ) - sys.exit() - except TransitionNotAllowed as err: - skipped_domain_entries.append(transition_domain_name) - logger.info( - f"""{termColors.FAIL} - Unable to change state for {transition_domain_name} - TRANSITION NOT ALLOWED error message (internal): - {err} - ----------SKIPPING----------""" - ) + def update_domain_status(self, + transition_domain:TransitionDomain, + target_domain:Domain, + debug_on:bool) -> bool: + """Given a transition domain that matches an existing domain, + updates the existing domain object with that status of + the transition domain. + Returns TRUE if an update was made. FALSE if the states + matched and no update was made""" + transition_domain_status = transition_domain.status + existing_status = target_domain.state + if transition_domain_status != existing_status: + if ( + transition_domain_status + == TransitionDomain.StatusChoices.ON_HOLD + ): + target_domain.place_client_hold(ignoreEPP=True) + else: + target_domain.revert_client_hold(ignoreEPP=True) + target_domain.save() # DEBUG: - if debug_on or debug_max_entries_to_parse > 0: - if ( - total_rows_parsed >= debug_max_entries_to_parse - and debug_max_entries_to_parse != 0 - ): - logger.info( - f"""{termColors.YELLOW} - ----PARSE LIMIT REACHED. HALTING PARSER.---- - {termColors.ENDC} - """ - ) - break + if debug_on: + logger.info( + f"""{termColors.YELLOW} + >> Updated {target_domain.name} state from + '{existing_status}' to '{target_domain.state}' + (no domain invitation entry added) + {termColors.ENDC}""" + ) - Domain.objects.bulk_create(domains_to_create) - DomainInvitation.objects.bulk_create(domain_invitations_to_create) + def print_summary_of_findings(self, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + skipped_domain_entries, + skipped_domain_invitations, + debug_on): + """Prints to terminal a summary of findings from + transferring transition domains to domains""" total_new_entries = len(domains_to_create) total_updated_domain_entries = len(updated_domain_entries) @@ -275,3 +157,150 @@ class Command(BaseCommand): {termColors.ENDC} """ ) + + def handle( + self, + **options, + ): + """Parse entries in TransitionDomain table + and create (or update) corresponding entries in the + Domain and DomainInvitation tables.""" + + # grab command line arguments and store locally... + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + + self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + + # domains to ADD + domains_to_create = [] + domain_invitations_to_create = [] + # domains we UPDATED + updated_domain_entries = [] + # domains we SKIPPED + skipped_domain_entries = [] + # domainInvitations we SKIPPED + skipped_domain_invitations = [] + # if we are limiting our parse (for testing purposes, keep + # track of total rows parsed) + total_rows_parsed = 0 + + logger.info( + f"""{termColors.OKGREEN} + ========================== + Beginning Data Transfer + ========================== + {termColors.ENDC}""" + ) + + for transition_domain in TransitionDomain.objects.all(): + transition_domain_name = transition_domain.domain_name + transition_domain_status = transition_domain.status + transition_domain_email = transition_domain.username + + # Check for existing domain entry + try: + # DEBUG: + if debug_on: + logger.info( + f"""{termColors.OKCYAN} + Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} + {termColors.ENDC}""" # noqa + ) + + # get the existing domain + target_domain = Domain.objects.get(name=transition_domain_name) + # DEBUG: + if debug_on: + logger.info( + f"""{termColors.YELLOW} + > Found existing domain entry for: {transition_domain_name}, {target_domain.state} + {termColors.ENDC}""" # noqa + ) + + # for existing entry, update the status to + # the transition domain status + update_made = self.update_domain_status(transition_domain, target_domain, debug_on) + if update_made: + updated_domain_entries.append(transition_domain.name) + + except Domain.DoesNotExist: + already_in_to_create = next( + (x for x in domains_to_create if x.name == transition_domain_name), + None, + ) + if already_in_to_create: + # DEBUG: + if debug_on: + logger.info( + f"""{termColors.YELLOW} + Duplicate Detected: {transition_domain_name}. + Cannot add duplicate entry for another username. + Violates Unique Key constraint. + {termColors.ENDC}""" + ) + else: + # no matching entry, make one + new_entry = Domain( + name=transition_domain_name, state=transition_domain_status + ) + domains_to_create.append(new_entry) + + if transition_domain_email: + new_domain_invitation = DomainInvitation( + email=transition_domain_email.lower(), domain=new_entry + ) + domain_invitations_to_create.append(new_domain_invitation) + else: + logger.info( + f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" # noqa + f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" + ) + skipped_domain_invitations.append(transition_domain_name) + + # DEBUG: + if debug_on: + logger.info( + f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}" # noqa + ) + except Domain.MultipleObjectsReturned: + logger.warning( + f""" + {termColors.FAIL} + !!! ERROR: duplicate entries exist in the + Domain table for domain: + {transition_domain_name} + ----------TERMINATING----------""" + ) + sys.exit() + except TransitionNotAllowed as err: + skipped_domain_entries.append(transition_domain_name) + logger.warning( + f"""{termColors.FAIL} + Unable to change state for {transition_domain_name} + TRANSITION NOT ALLOWED error message (internal): + {err} + ----------SKIPPING----------""" + ) + + # Check parse limit + if debug_max_entries_to_parse > 0 and total_rows_parsed >= debug_max_entries_to_parse: + logger.info( + f"""{termColors.YELLOW} + ----PARSE LIMIT REACHED. HALTING PARSER.---- + {termColors.ENDC} + """ + ) + break + + Domain.objects.bulk_create(domains_to_create) + DomainInvitation.objects.bulk_create(domain_invitations_to_create) + + self.print_summary_of_findings(domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + skipped_domain_entries, + skipped_domain_invitations, + debug_on) From 775ed5f9d2c8acbf1e97f4af5ed5fda16f4dcb33 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 11 Oct 2023 16:13:14 -0600 Subject: [PATCH 10/13] Refactored handle() to be less complex, satisfied linter Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 193 ++++++++++-------- 1 file changed, 105 insertions(+), 88 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 0e27bf20d..773825299 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -51,29 +51,37 @@ class Command(BaseCommand): ): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" - if debug_on: - logger.info( - f"""{termColors.OKCYAN} - ----------DEBUG MODE ON---------- - Detailed print statements activated. - {termColors.ENDC} - """ - ) - if debug_max_entries_to_parse > 0: - logger.info( - f"""{termColors.OKCYAN} - ----------LIMITER ON---------- - Parsing of entries will be limited to - {debug_max_entries_to_parse} lines per file.") - Detailed print statements activated. - {termColors.ENDC} - """ - ) + self.print_debug( + debug_on, + f"""{termColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {termColors.ENDC} + """, + ) + self.print_debug( + debug_max_entries_to_parse > 0, + f"""{termColors.OKCYAN} + ----------LIMITER ON---------- + Parsing of entries will be limited to + {debug_max_entries_to_parse} lines per file.") + Detailed print statements activated. + {termColors.ENDC} + """, + ) - def update_domain_status(self, - transition_domain:TransitionDomain, - target_domain:Domain, - debug_on:bool) -> bool: + def print_debug(self, print_condition: bool, print_statement: str): + """This function reduces complexity of debug statements + in other functions. + It uses the logger to write the given print_statement to the + terminal if print_condition is TRUE""" + # DEBUG: + if print_condition: + logger.info(print_statement) + + def update_domain_status( + self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool + ) -> bool: """Given a transition domain that matches an existing domain, updates the existing domain object with that status of the transition domain. @@ -83,31 +91,33 @@ class Command(BaseCommand): transition_domain_status = transition_domain.status existing_status = target_domain.state if transition_domain_status != existing_status: - if ( - transition_domain_status - == TransitionDomain.StatusChoices.ON_HOLD - ): + if transition_domain_status == TransitionDomain.StatusChoices.ON_HOLD: target_domain.place_client_hold(ignoreEPP=True) else: target_domain.revert_client_hold(ignoreEPP=True) target_domain.save() - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - >> Updated {target_domain.name} state from - '{existing_status}' to '{target_domain.state}' - (no domain invitation entry added) - {termColors.ENDC}""" - ) - def print_summary_of_findings(self, - domains_to_create, - updated_domain_entries, - domain_invitations_to_create, - skipped_domain_entries, - skipped_domain_invitations, - debug_on): + # DEBUG: + self.print_debug( + debug_on, + f"""{termColors.YELLOW} + >> Updated {target_domain.name} state from + '{existing_status}' to '{target_domain.state}' + (no domain invitation entry added) + {termColors.ENDC}""", + ) + return True + return False + + def print_summary_of_findings( + self, + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + skipped_domain_entries, + skipped_domain_invitations, + debug_on, + ): """Prints to terminal a summary of findings from transferring transition domains to domains""" @@ -119,7 +129,7 @@ class Command(BaseCommand): f"""{termColors.OKGREEN} ============= FINISHED =============== Created {total_new_entries} transition domain entries, - updated {total_updated_domain_entries} transition domain entries + Updated {total_updated_domain_entries} transition domain entries Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) @@ -144,19 +154,19 @@ class Command(BaseCommand): ) # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} + self.print_debug( + debug_on, + f"""{termColors.YELLOW} - Created Domains: - {domains_to_create} + Created Domains: + {domains_to_create} - Updated Domains: - {updated_domain_entries} + Updated Domains: + {updated_domain_entries} - {termColors.ENDC} - """ - ) + {termColors.ENDC} + """, + ) def handle( self, @@ -203,28 +213,31 @@ class Command(BaseCommand): # Check for existing domain entry try: # DEBUG: - if debug_on: - logger.info( - f"""{termColors.OKCYAN} - Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} - {termColors.ENDC}""" # noqa - ) - + self.print_debug( + debug_on, + f"""{termColors.OKCYAN} + Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} + {termColors.ENDC}""", # noqa + ) + # get the existing domain target_domain = Domain.objects.get(name=transition_domain_name) + # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - > Found existing domain entry for: {transition_domain_name}, {target_domain.state} - {termColors.ENDC}""" # noqa - ) + self.print_debug( + debug_on, + f"""{termColors.YELLOW} + > Found existing domain entry for: {transition_domain_name}, {target_domain.state} + {termColors.ENDC}""", # noqa + ) # for existing entry, update the status to # the transition domain status - update_made = self.update_domain_status(transition_domain, target_domain, debug_on) + update_made = self.update_domain_status( + transition_domain, target_domain, debug_on + ) if update_made: - updated_domain_entries.append(transition_domain.name) + updated_domain_entries.append(transition_domain.domain_name) except Domain.DoesNotExist: already_in_to_create = next( @@ -232,15 +245,14 @@ class Command(BaseCommand): None, ) if already_in_to_create: - # DEBUG: - if debug_on: - logger.info( - f"""{termColors.YELLOW} - Duplicate Detected: {transition_domain_name}. - Cannot add duplicate entry for another username. - Violates Unique Key constraint. - {termColors.ENDC}""" - ) + self.print_debug( + debug_on, + f"""{termColors.YELLOW} + Duplicate Detected: {transition_domain_name}. + Cannot add duplicate entry for another username. + Violates Unique Key constraint. + {termColors.ENDC}""", + ) else: # no matching entry, make one new_entry = Domain( @@ -261,10 +273,10 @@ class Command(BaseCommand): skipped_domain_invitations.append(transition_domain_name) # DEBUG: - if debug_on: - logger.info( - f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}" # noqa - ) + self.print_debug( + debug_on, + f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}", # noqa + ) except Domain.MultipleObjectsReturned: logger.warning( f""" @@ -286,7 +298,10 @@ class Command(BaseCommand): ) # Check parse limit - if debug_max_entries_to_parse > 0 and total_rows_parsed >= debug_max_entries_to_parse: + if ( + debug_max_entries_to_parse > 0 + and total_rows_parsed >= debug_max_entries_to_parse + ): logger.info( f"""{termColors.YELLOW} ----PARSE LIMIT REACHED. HALTING PARSER.---- @@ -298,9 +313,11 @@ class Command(BaseCommand): Domain.objects.bulk_create(domains_to_create) DomainInvitation.objects.bulk_create(domain_invitations_to_create) - self.print_summary_of_findings(domains_to_create, - updated_domain_entries, - domain_invitations_to_create, - skipped_domain_entries, - skipped_domain_invitations, - debug_on) + self.print_summary_of_findings( + domains_to_create, + updated_domain_entries, + domain_invitations_to_create, + skipped_domain_entries, + skipped_domain_invitations, + debug_on, + ) From 6382b14c6ef8460772578f85e571d641088160a3 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 12 Oct 2023 15:16:25 -0600 Subject: [PATCH 11/13] updated algorithm for domain invitations to allow multiple users per domain Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 209 ++++++++++++------ 1 file changed, 140 insertions(+), 69 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 773825299..97f9cd893 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -1,5 +1,3 @@ -"""Load domain invitations for existing domains and their contacts.""" - import logging import argparse import sys @@ -115,8 +113,7 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, - skipped_domain_invitations, - debug_on, + debug_on ): """Prints to terminal a summary of findings from transferring transition domains to domains""" @@ -144,6 +141,13 @@ class Command(BaseCommand): {termColors.ENDC} """ ) + + # determine domainInvitations we SKIPPED + skipped_domain_invitations = [] + for domain in domains_to_create: + skipped_domain_invitations.append(domain) + for domain_invite in domain_invitations_to_create: + if domain_invite.domain in skipped_domain_invitations: skipped_domain_invitations.remove(domain_invite.domain) if len(skipped_domain_invitations) > 0: logger.info( f"""{termColors.FAIL} @@ -168,6 +172,33 @@ class Command(BaseCommand): """, ) + def try_add_domain_invitation(self, + domain_email: str, + associated_domain: Domain) -> DomainInvitation: + """If no domain invitation exists for the given domain and + e-mail, create and return a new domain invitation object. + If one already exists, or if the email is invalid, return NONE""" + + # this exception should never happen, but adding it just in case + assert (associated_domain is None, "domain cannot be null for a Domain Invitation object!") + + # check that the given e-mail is valid + if domain_email is not None and domain_email != "": + # check that a domain invitation doesn't already + # exist for this e-mail / Domain pair + domain_email_already_in_domain_invites = DomainInvitation.objects.filter( + email = domain_email.lower(), + domain=associated_domain).exists() + if not domain_email_already_in_domain_invites: + # Create new domain invitation + new_domain_invitation = DomainInvitation( + email=domain_email.lower(), + domain=associated_domain + ) + return new_domain_invitation + return None + + def handle( self, **options, @@ -191,8 +222,6 @@ class Command(BaseCommand): updated_domain_entries = [] # domains we SKIPPED skipped_domain_entries = [] - # domainInvitations we SKIPPED - skipped_domain_invitations = [] # if we are limiting our parse (for testing purposes, keep # track of total rows parsed) total_rows_parsed = 0 @@ -210,92 +239,135 @@ class Command(BaseCommand): transition_domain_status = transition_domain.status transition_domain_email = transition_domain.username + # DEBUG: + self.print_debug( + debug_on, + f"""{termColors.OKCYAN} + Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} + {termColors.ENDC}""", # noqa + ) + + new_domain_invitation = None # Check for existing domain entry - try: - # DEBUG: - self.print_debug( - debug_on, - f"""{termColors.OKCYAN} - Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} - {termColors.ENDC}""", # noqa - ) + domain_exists = Domain.objects.filter( + name=transition_domain_name + ).exists() + if domain_exists: + try: + # get the existing domain + domain_to_update = Domain.objects.get(name=transition_domain_name) + # DEBUG: + self.print_debug( + debug_on, + f"""{termColors.YELLOW} + > Found existing entry in Domain table for: {transition_domain_name}, {domain_to_update.state} + {termColors.ENDC}""", # noqa + ) - # get the existing domain - target_domain = Domain.objects.get(name=transition_domain_name) + # for existing entry, update the status to + # the transition domain status + update_made = self.update_domain_status( + transition_domain, domain_to_update, debug_on + ) + if update_made: + # keep track of updated domains for data analysis purposes + updated_domain_entries.append(transition_domain.domain_name) - # DEBUG: - self.print_debug( - debug_on, - f"""{termColors.YELLOW} - > Found existing domain entry for: {transition_domain_name}, {target_domain.state} - {termColors.ENDC}""", # noqa - ) + # check if we need to add a domain invitation + # (eg. for a new user) + new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, domain_to_update) - # for existing entry, update the status to - # the transition domain status - update_made = self.update_domain_status( - transition_domain, target_domain, debug_on - ) - if update_made: - updated_domain_entries.append(transition_domain.domain_name) + except Domain.MultipleObjectsReturned: + # This exception was thrown once before during testing. + # While the circumstances that led to corrupt data in + # the domain table was a freak accident, and the possibility of it + # happening again is safe-guarded by a key constraint, + # better to keep an eye out for it since it would require + # immediate attention. + logger.warning( + f""" + {termColors.FAIL} + !!! ERROR: duplicate entries already exist in the + Domain table for the following domain: + {transition_domain_name} + + RECOMMENDATION: + This means the Domain table is corrupt. Please + check the Domain table data as there should be a key + constraint which prevents duplicate entries. - except Domain.DoesNotExist: - already_in_to_create = next( + ----------TERMINATING----------""" + ) + sys.exit() + except TransitionNotAllowed as err: + skipped_domain_entries.append(transition_domain_name) + logger.warning( + f"""{termColors.FAIL} + Unable to change state for {transition_domain_name} + + RECOMMENDATION: + This indicates there might have been changes to the + Domain model which were not accounted for in this + migration script. Please check state change rules + in the Domain model and ensure we are following the + correct state transition pathways. + + INTERNAL ERROR MESSAGE: + 'TRANSITION NOT ALLOWED' exception + {err} + ----------SKIPPING----------""" + ) + else: + # no entry was found in the domain table + # for the given domain. Create a new entry. + + # first see if we are already adding an entry for this domain. + # The unique key constraint does not allow duplicate domain entries + # even if there are different users. + existing_domain_in_to_create = next( (x for x in domains_to_create if x.name == transition_domain_name), None, ) - if already_in_to_create: + if existing_domain_in_to_create is not None: self.print_debug( debug_on, f"""{termColors.YELLOW} Duplicate Detected: {transition_domain_name}. Cannot add duplicate entry for another username. Violates Unique Key constraint. + + Checking for unique user e-mail for Domain Invitations... {termColors.ENDC}""", ) + new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, existing_domain_in_to_create) else: # no matching entry, make one - new_entry = Domain( - name=transition_domain_name, state=transition_domain_status + new_domain = Domain( + name=transition_domain_name, + state=transition_domain_status ) - domains_to_create.append(new_entry) - - if transition_domain_email: - new_domain_invitation = DomainInvitation( - email=transition_domain_email.lower(), domain=new_entry - ) - domain_invitations_to_create.append(new_domain_invitation) - else: - logger.info( - f"{termColors.FAIL} ! No e-mail found for domain: {new_entry}" # noqa - f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" - ) - skipped_domain_invitations.append(transition_domain_name) - + domains_to_create.append(new_domain) # DEBUG: self.print_debug( debug_on, - f"{termColors.OKCYAN} Adding domain AND domain invitation: {new_entry} {termColors.ENDC}", # noqa + f"{termColors.OKCYAN} Adding domain: {new_domain} {termColors.ENDC}" ) - except Domain.MultipleObjectsReturned: - logger.warning( - f""" - {termColors.FAIL} - !!! ERROR: duplicate entries exist in the - Domain table for domain: - {transition_domain_name} - ----------TERMINATING----------""" + new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, new_domain) + + if new_domain_invitation is None: + logger.info( + f"{termColors.YELLOW} ! No new e-mail detected !" # noqa + f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" ) - sys.exit() - except TransitionNotAllowed as err: - skipped_domain_entries.append(transition_domain_name) - logger.warning( - f"""{termColors.FAIL} - Unable to change state for {transition_domain_name} - TRANSITION NOT ALLOWED error message (internal): - {err} - ----------SKIPPING----------""" + else: + # DEBUG: + self.print_debug( + debug_on, + f"{termColors.OKCYAN} Adding domain invitation: {new_domain_invitation} {termColors.ENDC}", ) + domain_invitations_to_create.append(new_domain_invitation) + + # Check parse limit if ( @@ -318,6 +390,5 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, - skipped_domain_invitations, - debug_on, + debug_on ) From d617c2a3150249b4543e521d89518f156b351997 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 12 Oct 2023 15:32:59 -0600 Subject: [PATCH 12/13] linted Signed-off-by: CocoByte --- .../transfer_transition_domains_to_domains.py | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 97f9cd893..b98e8e2a9 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -113,7 +113,7 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, - debug_on + debug_on, ): """Prints to terminal a summary of findings from transferring transition domains to domains""" @@ -147,7 +147,8 @@ class Command(BaseCommand): for domain in domains_to_create: skipped_domain_invitations.append(domain) for domain_invite in domain_invitations_to_create: - if domain_invite.domain in skipped_domain_invitations: skipped_domain_invitations.remove(domain_invite.domain) + if domain_invite.domain in skipped_domain_invitations: + skipped_domain_invitations.remove(domain_invite.domain) if len(skipped_domain_invitations) > 0: logger.info( f"""{termColors.FAIL} @@ -172,33 +173,46 @@ class Command(BaseCommand): """, ) - def try_add_domain_invitation(self, - domain_email: str, - associated_domain: Domain) -> DomainInvitation: + def try_add_domain_invitation( + self, domain_email: str, associated_domain: Domain + ) -> DomainInvitation | None: """If no domain invitation exists for the given domain and e-mail, create and return a new domain invitation object. If one already exists, or if the email is invalid, return NONE""" - # this exception should never happen, but adding it just in case - assert (associated_domain is None, "domain cannot be null for a Domain Invitation object!") - + # this should never happen, but adding it just in case + if associated_domain is None: + logger.warning( + f""" + {termColors.FAIL} + !!! ERROR: Domain cannot be null for a + Domain Invitation object! + + RECOMMENDATION: + Somehow, an empty domain object is + being passed to the subroutine in charge + of making domain invitations. Walk through + the code to see what is amiss. + + ----------TERMINATING----------""" + ) + sys.exit() + # check that the given e-mail is valid if domain_email is not None and domain_email != "": - # check that a domain invitation doesn't already + # check that a domain invitation doesn't already # exist for this e-mail / Domain pair domain_email_already_in_domain_invites = DomainInvitation.objects.filter( - email = domain_email.lower(), - domain=associated_domain).exists() + email=domain_email.lower(), domain=associated_domain + ).exists() if not domain_email_already_in_domain_invites: # Create new domain invitation new_domain_invitation = DomainInvitation( - email=domain_email.lower(), - domain=associated_domain + email=domain_email.lower(), domain=associated_domain ) return new_domain_invitation return None - def handle( self, **options, @@ -249,9 +263,7 @@ class Command(BaseCommand): new_domain_invitation = None # Check for existing domain entry - domain_exists = Domain.objects.filter( - name=transition_domain_name - ).exists() + domain_exists = Domain.objects.filter(name=transition_domain_name).exists() if domain_exists: try: # get the existing domain @@ -273,9 +285,11 @@ class Command(BaseCommand): # keep track of updated domains for data analysis purposes updated_domain_entries.append(transition_domain.domain_name) - # check if we need to add a domain invitation + # check if we need to add a domain invitation # (eg. for a new user) - new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, domain_to_update) + new_domain_invitation = self.try_add_domain_invitation( + transition_domain_email, domain_to_update + ) except Domain.MultipleObjectsReturned: # This exception was thrown once before during testing. @@ -290,7 +304,7 @@ class Command(BaseCommand): !!! ERROR: duplicate entries already exist in the Domain table for the following domain: {transition_domain_name} - + RECOMMENDATION: This means the Domain table is corrupt. Please check the Domain table data as there should be a key @@ -304,7 +318,7 @@ class Command(BaseCommand): logger.warning( f"""{termColors.FAIL} Unable to change state for {transition_domain_name} - + RECOMMENDATION: This indicates there might have been changes to the Domain model which were not accounted for in this @@ -339,21 +353,24 @@ class Command(BaseCommand): Checking for unique user e-mail for Domain Invitations... {termColors.ENDC}""", ) - new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, existing_domain_in_to_create) + new_domain_invitation = self.try_add_domain_invitation( + transition_domain_email, existing_domain_in_to_create + ) else: # no matching entry, make one new_domain = Domain( - name=transition_domain_name, - state=transition_domain_status + name=transition_domain_name, state=transition_domain_status ) domains_to_create.append(new_domain) # DEBUG: self.print_debug( debug_on, - f"{termColors.OKCYAN} Adding domain: {new_domain} {termColors.ENDC}" + f"{termColors.OKCYAN} Adding domain: {new_domain} {termColors.ENDC}", # noqa ) - new_domain_invitation = self.try_add_domain_invitation(transition_domain_email, new_domain) - + new_domain_invitation = self.try_add_domain_invitation( + transition_domain_email, new_domain + ) + if new_domain_invitation is None: logger.info( f"{termColors.YELLOW} ! No new e-mail detected !" # noqa @@ -363,13 +380,11 @@ class Command(BaseCommand): # DEBUG: self.print_debug( debug_on, - f"{termColors.OKCYAN} Adding domain invitation: {new_domain_invitation} {termColors.ENDC}", + f"{termColors.OKCYAN} Adding domain invitation: {new_domain_invitation} {termColors.ENDC}", # noqa ) domain_invitations_to_create.append(new_domain_invitation) - - - # Check parse limit + # Check parse limit and exit loop if parse limit has been reached if ( debug_max_entries_to_parse > 0 and total_rows_parsed >= debug_max_entries_to_parse @@ -390,5 +405,5 @@ class Command(BaseCommand): updated_domain_entries, domain_invitations_to_create, skipped_domain_entries, - debug_on + debug_on, ) From 42b0427e05f8cb33e68c60cbffc05b546b2b1455 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 12 Oct 2023 15:35:21 -0600 Subject: [PATCH 13/13] resequenced migration files Signed-off-by: CocoByte --- ...domain_status.py => 0039_alter_transitiondomain_status.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0038_alter_transitiondomain_status.py => 0039_alter_transitiondomain_status.py} (85%) diff --git a/src/registrar/migrations/0038_alter_transitiondomain_status.py b/src/registrar/migrations/0039_alter_transitiondomain_status.py similarity index 85% rename from src/registrar/migrations/0038_alter_transitiondomain_status.py rename to src/registrar/migrations/0039_alter_transitiondomain_status.py index 5e83e7076..e562f57c1 100644 --- a/src/registrar/migrations/0038_alter_transitiondomain_status.py +++ b/src/registrar/migrations/0039_alter_transitiondomain_status.py @@ -1,11 +1,11 @@ -# Generated by Django 4.2.1 on 2023-10-10 17:38 +# Generated by Django 4.2.1 on 2023-10-12 21:34 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("registrar", "0037_create_groups_v01"), + ("registrar", "0038_create_groups_v02"), ] operations = [