From 02ec4d7bae7127fe39ed3db883ab4f194ac229a0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 19:45:16 -0600 Subject: [PATCH] Renamed main script and added test script (to separate concerns of testing vs combining transition domain migration scripts). Added test data files. --- .../commands/full_domain_migrations.py | 532 ++++++++++++++++++ src/registrar/tests/data/test_contacts.txt | 7 + .../tests/data/test_domain_contacts.txt | 20 + .../tests/data/test_domain_statuses.txt | 6 + ...ransition_domain_migrations_with_logins.py | 42 ++ 5 files changed, 607 insertions(+) create mode 100644 src/registrar/management/commands/full_domain_migrations.py create mode 100644 src/registrar/tests/data/test_contacts.txt create mode 100644 src/registrar/tests/data/test_domain_contacts.txt create mode 100644 src/registrar/tests/data/test_domain_statuses.txt create mode 100644 src/registrar/tests/test_transition_domain_migrations_with_logins.py diff --git a/src/registrar/management/commands/full_domain_migrations.py b/src/registrar/management/commands/full_domain_migrations.py new file mode 100644 index 000000000..7e47e8075 --- /dev/null +++ b/src/registrar/management/commands/full_domain_migrations.py @@ -0,0 +1,532 @@ +"""Data migration: + 1 - generates a report of data integrity across all + transition domain related tables + 2 - allows users to run all migration scripts for + transition domain data +""" + +import logging +import argparse +import sys +import os + +from django.test import Client + +from django_fsm import TransitionNotAllowed # type: ignore + +from django.core.management import BaseCommand + +from registrar.models import ( + Domain, + DomainInformation, + DomainInvitation, + TransitionDomain, + User, +) + +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper +) + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = """ """ + + def add_arguments(self, parser): + """ + OPTIONAL ARGUMENTS: + --runLoaders + A boolean (default to true), which triggers running + all scripts (in sequence) for transition domain migrations + + --triggerLogins + A boolean (default to true), which triggers running + simulations of user logins for each user in domain invitation + + --loaderDirectory + The location of the files used for load_transition_domain migration script + EXAMPLE USAGE: + > --loaderDirectory /app/tmp + + --loaderFilenames + The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + EXAMPLE USAGE: + > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information + + --sep + Delimiter for the loaders to correctly parse the given text files. + (usually this can remain at default value of |) + + --debug + A boolean (default to true), which activates additional print statements + + --limitParse + Used by the loaders (load_transition_domain) to set the limit for the + number of data entries to insert. Set to 0 (or just don't use this + argument) to parse every entry. This was provided primarily for testing + purposes + + --resetTable + Used by the loaders to trigger a prompt for deleting all table entries. + Useful for testing purposes, but USE WITH CAUTION + """ + + parser.add_argument("--runLoaders", + help="Runs all scripts (in sequence) for transition domain migrations", + action=argparse.BooleanOptionalAction) + + parser.add_argument("--triggerLogins", + help="Simulates a user login for each user in domain invitation", + action=argparse.BooleanOptionalAction) + + # The following file arguments have default values for running in the sandbox + parser.add_argument( + "--loaderDirectory", + default="migrationData", + help="The location of the files used for load_transition_domain migration script" + ) + parser.add_argument( + "--loaderFilenames", + default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information""" + ) + + parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") + + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", default=0, help="Sets max number of entries to load" + ) + + parser.add_argument( + "--resetTable", + help="Deletes all data in the TransitionDomain table", + action=argparse.BooleanOptionalAction, + ) + + + + def compare_tables(self, debug_on: bool): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + + Produces the following report (printed to the terminal): + #1 - Print any domains that exist in the transition_domain table + but not in their corresponding domain, domain information or + domain invitation tables. + #2 - Print which table this domain is missing from + #3- Check for duplicate entries in domain or + domain_information tables and print which are + duplicates and in which tables + """ + + logger.info( + f"""{TerminalColors.OKCYAN} + ============= BEGINNING ANALYSIS =============== + {TerminalColors.ENDC} + """ + ) + + #TODO: would filteredRelation be faster? + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] + for transition_domain in TransitionDomain.objects.all():# DEBUG: + transition_domain_name = transition_domain.domain_name + transition_domain_email = transition_domain.username + + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN}Checking: {transition_domain_name} {TerminalColors.ENDC}", # noqa + ) + + # Check Domain table + matching_domains = Domain.objects.filter(name=transition_domain_name) + # Check Domain Information table + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + # Check Domain Invitation table + matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), + domain__name=transition_domain_name) + + if len(matching_domains) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") + missing_domains.append(transition_domain_name) + elif len(matching_domains) > 1: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") + duplicate_domains.append(transition_domain_name) + if len(matching_domain_informations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") + missing_domain_informations.append(transition_domain_name) + if len(matching_domain_invitations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") + missing_domain_invites.append(transition_domain_name) + + total_missing_domains = len(missing_domains) + total_duplicate_domains = len(duplicate_domains) + total_missing_domain_informations = len(missing_domain_informations) + total_missing_domain_invitations = len(missing_domain_invites) + + missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) + duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) + missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) + missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) + + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED ANALYSIS =============== + + {total_missing_domains} Missing Domains: + (These are transition domains that are missing from the Domain Table) + {TerminalColors.YELLOW}{missing_domains_as_string}{TerminalColors.OKGREEN} + + {total_duplicate_domains} Duplicate Domains: + (These are transition domains which have duplicate entries in the Domain Table) + {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_informations} Domain Information Entries missing: + (These are transition domains which have no entries in the Domain Information Table) + {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_invitations} Domain Invitations missing: + (These are transition domains which have no entires in the Domain Invitation Table) + {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} + {TerminalColors.ENDC} + """ + ) + + def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: + """Prompts the user to inspect the given terminal command string + and asks if they wish to execute it. If the user responds (y), + execute the command""" + + # Allow the user to inspect the command string + # and ask if they wish to proceed + proceed_execution = TerminalHelper.query_yes_no( + f"""{TerminalColors.OKCYAN} + ===================================================== + {prompt_title} + ===================================================== + *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** + + {command_string} + {TerminalColors.FAIL} + Proceed? (Y = proceed, N = skip) + {TerminalColors.ENDC}""" + ) + + # If the user decided to proceed executing the command, + # run the command for loading transition domains. + # Otherwise, exit this subroutine. + if not proceed_execution: + sys.exit() + + self.execute_command(command_string) + + return True + + def execute_command(self, command_string:str): + """Executes the given command string""" + + logger.info(f"""{TerminalColors.OKCYAN} + ==== EXECUTING... ==== + {TerminalColors.ENDC}""") + os.system(f"{command_string}") + + def run_load_transition_domain_script(self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int): + """Runs the load_transition_domain script""" + # Create the command string + command_string = "./manage.py load_transition_domain " + command_string += file_location+domain_contacts_filename + " " + command_string += file_location+contacts_filename + " " + command_string += file_location+domain_statuses_filename + " " + if sep is not None and sep != "|": + command_string += f"--sep {sep} " + if reset_table: + command_string += "--resetTable " + if debug_on: + command_string += "--debug " + if debug_max_entries_to_parse > 0: + command_string += f"--limitParse {debug_max_entries_to_parse} " + + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running load_transition_domain script") + return + self.execute_command(command_string) + + + def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): + """Runs the transfer_transition_domains_to_domains script""" + # Create the command string + command_string = "./manage.py transfer_transition_domains_to_domains " + if debug_on: + command_string += "--debug " + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + return + self.execute_command(command_string) + + + def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): + """Runs the send_domain_invitations script""" + # Create the command string... + command_string = "./manage.py send_domain_invitations -s" + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running send_domain_invitations script") + return + self.execute_command(command_string) + + + def run_migration_scripts(self, + prompts_enabled: bool, + options): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" + + # Get arguments + sep = options.get("sep") + reset_table = options.get("resetTable") + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) + + # Grab filepath information from the arguments + file_location = options.get("loaderDirectory")+"/" + filenames = options.get("loaderFilenames").split() + if len(filenames) < 3: + filenames_as_string = "{}".format(", ".join(map(str, filenames))) + logger.info(f""" + {TerminalColors.FAIL} + --loaderFilenames expected 3 filenames to follow it, + but only {len(filenames)} were given: + {filenames_as_string} + + PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN + ============= TERMINATING ============= + {TerminalColors.ENDC} + """) + sys.exit() + domain_contacts_filename = filenames[0] + contacts_filename = filenames[1] + domain_statuses_filename = filenames[2] + + if prompts_enabled: + # Allow the user to inspect the filepath + # data given in the arguments, and prompt + # the user to verify this info before proceeding + files_are_correct = TerminalHelper.query_yes_no( + f""" + {TerminalColors.OKCYAN} + *** IMPORTANT: VERIFY THE FOLLOWING *** + + The migration scripts are looking in directory.... + {file_location} + + ....for the following files: + - domain contacts: {domain_contacts_filename} + - contacts: {contacts_filename} + - domain statuses: {domain_statuses_filename} + + {TerminalColors.FAIL} + Does this look correct?{TerminalColors.ENDC}""" + ) + + # If the user rejected the filepath information + # as incorrect, prompt the user to provide + # correct file inputs in their original command + # prompt and exit this subroutine + if not files_are_correct: + logger.info(f""" + {TerminalColors.YELLOW} + PLEASE Re-Run the script with the correct file location and filenames: + + EXAMPLE: + docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt + + """) + return + + # Proceed executing the migration scripts + self.run_load_transition_domain_script(file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse) + self.run_transfer_script(debug_on, prompts_enabled) + + + def simulate_user_logins(self, debug_on): + """Simulates logins for users (this will add + Domain Information objects to our tables)""" + + logger.info(f"" + f"{TerminalColors.OKCYAN}" + f"================== SIMULATING LOGINS ==================" + f"{TerminalColors.ENDC}") + + # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff + # #DEBUG: + # TerminalHelper.print_conditional(debug_on, + # f"{TerminalColors.OKCYAN}" + # f"Processing invite: {invite}" + # f"{TerminalColors.ENDC}") + # # get a user with this email address + # user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + # #DEBUG: + # TerminalHelper.print_conditional(user_created, + # f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") + # TerminalHelper.print_conditional(debug_on, + # f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") + # user.first_login() + # if user_created: + # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + # user.delete() + + + def handle( + self, + **options, + ): + """ + Does the following; + 1 - run loader scripts + 2 - simulate logins + 3 - send domain invitations (Emails should be sent to the appropriate users + note that all moved domains should now be accessible + on django admin for an analyst) + 4 - analyze the data for transition domains + and generate a report + """ + + # SETUP + # Grab all arguments relevant to + # orchestrating which parts of this script + # should execute. Print some indicators to + # the terminal so the user knows what is + # enabled. + debug_on = options.get("debug") + prompts_enabled = debug_on #TODO: add as argument? + run_loaders_enabled = options.get("runLoaders") + simulate_user_login_enabled = options.get("triggerLogins") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} + ----------RUNNING LOADERS ON---------- + All migration scripts will be run before + analyzing the data. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} + ----------TRIGGER LOGINS ON---------- + Will be simulating user logins + {TerminalColors.ENDC} + """ + ) + # If a user decides to run all migration + # scripts, they may or may not wish to + # proceed with analysis of the data depending + # on the results of the migration. + # Provide a breakpoint for them to decide + # whether to continue or not. + # The same will happen if simulating user + # logins (to allow users to run only that + # portion of the script if desired) + prompt_continuation_of_analysis = False + + # STEP 1 -- RUN LOADERS + # Run migration scripts if specified by user + if run_loaders_enabled: + self.run_migration_scripts(options, prompts_enabled) + prompt_continuation_of_analysis = True + + # STEP 2 -- SIMULATE LOGINS + # Simulate user login for each user in domain + # invitation if specified by user OR if running + # migration scripts. + # (NOTE: Although users can choose to run login + # simulations separately (for testing purposes), + # if we are running all migration scripts, we should + # automatically execute this as the final step + # to ensure Domain Information objects get added + # to the database.) + if run_loaders_enabled: + if prompts_enabled: + simulate_user_login_enabled = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with simulating user logins? + {TerminalColors.ENDC}""" + ) + if not simulate_user_login_enabled: + return + self.simulate_user_logins(debug_on) + prompt_continuation_of_analysis = True + + # STEP 3 -- SEND INVITES + if prompts_enabled: + proceed_with_sending_invites = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with sending user invites? + {TerminalColors.ENDC}""" + ) + if not proceed_with_sending_invites: + return + self.run_send_invites_script(debug_on) + prompt_continuation_of_analysis = True + + # STEP 4 -- ANALYZE TABLES & GENERATE REPORT + # Analyze tables for corrupt data... + if prompt_continuation_of_analysis & prompts_enabled: + # ^ (only prompt if we ran steps 1 and/or 2) + analyze_tables = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with table analysis? + {TerminalColors.ENDC}""" + ) + if not analyze_tables: + return + self.compare_tables(debug_on) diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt new file mode 100644 index 000000000..dec8f6816 --- /dev/null +++ b/src/registrar/tests/data/test_contacts.txt @@ -0,0 +1,7 @@ +TESTUSER|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||testuser@gmail.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| +RJD1|52545_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||agustina.wyman7@test.com|GSA|VERISIGN|ctldbatch|2021-06-29T18:53:09Z|VERISIGN|ctldbatch|2021-06-29T18:58:08Z| +JAKING|52555_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||susy.martin4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T15:38:10Z| +JBOONE|52556_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||stephania.winters4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T18:28:09Z| +MKELLEY|52557_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||alexandra.bobbitt5@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-08-02T22:13:09Z| +CWILSON|52562_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||jospeh.mcdowell3@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:33:09Z| +LMCCADE|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||reginald.ratcliff4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt new file mode 100644 index 000000000..c25181c28 --- /dev/null +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -0,0 +1,20 @@ +Anomaly.gov|ANOMALY|tech +TestDomain.gov|TESTUSER|admin +NEHRP.GOV|RJD1|admin +NEHRP.GOV|JAKING|tech +NEHRP.GOV|JBOONE|billing +NELSONCOUNTY-VA.GOV|MKELLEY|admin +NELSONCOUNTY-VA.GOV|CWILSON|billing +NELSONCOUNTY-VA.GOV|LMCCADE|tech +NELSONVILLENY.GOV|MBOWMAN|tech +NELSONVILLENY.GOV|PMINNERS|billing +NELSONVILLENY.GOV|CWINWARD|admin +NEMI.GOV|DJS1|tech +NEMI.GOV|EREAD|admin +NEMI.GOV|CHOPKINS|billing +NEPA.GOV|BPAUWELS|admin +NEPA.GOV|CRIBEIRO|billing +NEPA.GOV|DKAUFMAN1|tech +NERSC.GOV|TONEILL|admin +NERSC.GOV|JGERSTLE|billing +NERSC.GOV|RSTROMSNESS|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt new file mode 100644 index 000000000..60d526f2d --- /dev/null +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -0,0 +1,6 @@ +Anomaly.gov|muahaha| +TestDomain.gov|ok| +NEHRP.GOV|serverHold| +NELSONCOUNTY-VA.GOV|Hold| +NEMI.GOV|clientHold| +NERSC.GOV|ok| \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations_with_logins.py b/src/registrar/tests/test_transition_domain_migrations_with_logins.py new file mode 100644 index 000000000..2b3d7b6e7 --- /dev/null +++ b/src/registrar/tests/test_transition_domain_migrations_with_logins.py @@ -0,0 +1,42 @@ +from django.test import TestCase +from django.db.utils import IntegrityError +from unittest.mock import patch + +from registrar.models import ( + Contact, + DomainApplication, + DomainInformation, + User, + Website, + Domain, + DraftDomain, + DomainInvitation, + UserDomainRole, +) + +import boto3_mocking # type: ignore +from .common import MockSESClient, less_console_noise, completed_application +from django_fsm import TransitionNotAllowed + +boto3_mocking.clients.register_handler("sesv2", MockSESClient) + +@boto3_mocking.patching +class TestInvitations(TestCase): + + """Test the retrieval of invitations.""" + + def setUp(self): + self.domain, _ = Domain.objects.get_or_create(name="igorville.gov") + self.email = "mayor@igorville.gov" + self.invitation, _ = DomainInvitation.objects.get_or_create( + email=self.email, domain=self.domain + ) + self.user, _ = User.objects.get_or_create(email=self.email) + + # clean out the roles each time + UserDomainRole.objects.all().delete() + + def test_user_logins(self): + """A new user's first_login callback retrieves their invitations.""" + self.user.first_login() + self.assertTrue(UserDomainRole.objects.get(user=self.user, domain=self.domain))