From 43e619dc1a541f70097f52481ec82fa46b3e38b4 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 19 Oct 2023 17:52:49 -0600 Subject: [PATCH 01/49] scaffolded script Signed-off-by: CocoByte --- .../commands/test_domain_migration.py | 107 ++++++++++++++++++ .../management/commands/utility/enums.py | 0 2 files changed, 107 insertions(+) create mode 100644 src/registrar/management/commands/test_domain_migration.py create mode 100644 src/registrar/management/commands/utility/enums.py diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py new file mode 100644 index 000000000..7bdda7643 --- /dev/null +++ b/src/registrar/management/commands/test_domain_migration.py @@ -0,0 +1,107 @@ +import logging +import argparse +import sys + +from django_fsm import TransitionNotAllowed # type: ignore + +from django.core.management import BaseCommand + +from registrar.models import TransitionDomain +from registrar.models import Domain +from registrar.models import DomainInvitation + +logger = logging.getLogger(__name__) + + +class termColors: + """Colors for terminal outputs + (makes reading the logs WAY easier)""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + YELLOW = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + BackgroundLightYellow = "\033[103m" + + +class Command(BaseCommand): + help = """ """ + + def add_arguments(self, parser): + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", + default=0, + help="Sets max number of entries to load, set to 0 to load all entries", + ) + + def print_debug_mode_statements( + self, debug_on: bool, debug_max_entries_to_parse: int + ): + """Prints additional terminal statements to indicate if --debug + or --limitParse are in use""" + self.print_debug( + debug_on, + f"""{termColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {termColors.ENDC} + """, + ) + self.print_debug( + debug_max_entries_to_parse > 0, + f"""{termColors.OKCYAN} + ----------LIMITER ON---------- + Parsing of entries will be limited to + {debug_max_entries_to_parse} lines per file.") + Detailed print statements activated. + {termColors.ENDC} + """, + ) + + def print_debug(self, print_condition: bool, print_statement: str): + """This function reduces complexity of debug statements + in other functions. + It uses the logger to write the given print_statement to the + terminal if print_condition is TRUE""" + # DEBUG: + if print_condition: + logger.info(print_statement) + + + def handle( + self, + **options, + ): + """ + Do a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + + It should: + - Print any domains that exist in the transition_domain table + but not in their corresponding domain, domain information or + domain invitation tables. + - Print which table this domain is missing from + - Check for duplicate entries in domain or + domain_information tables and print which are + duplicates and in which tables + + (ONLY RUNS with full script option) + - Emails should be sent to the appropriate users + note that all moved domains should now be accessible + on django admin for an analyst + + OPTIONS: + -- (run all other scripts: + 1 - imports for trans domains + 2 - transfer to domain & domain invitation + 3 - send domain invite) + ** Triggers table reset ** + """ + diff --git a/src/registrar/management/commands/utility/enums.py b/src/registrar/management/commands/utility/enums.py new file mode 100644 index 000000000..e69de29bb From 70388d6185c9d725caa96611ab362b8a2cd78a66 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:15:41 -0600 Subject: [PATCH 02/49] Update data_migration.md --- docs/operations/data_migration.md | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 192db0db8..0a446c7ae 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -84,6 +84,82 @@ FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain na FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses +## Load migration data onto a production or sandbox environment +**WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. +Do not use this method to store data you want to keep around permanently. + +### STEP 1: Use scp to transfer data +CloudFoundry supports scp as means of transferring data locally to our environment. If you are dealing with a batch of files, try sending across a tar.gz and unpacking that. + +**Login to Cloud.gov** + +```bash +cf login -a api.fr.cloud.gov --sso +``` + +**Target your workspace** + +```bash +cf target -o cisa-dotgov -s {SANDBOX_NAME} +``` +*SANDBOX_NAME* - Name of your sandbox, ex: za or ab + +**Run the scp command** + +Use the following command to transfer the desired file: +```shell +scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HERE} --guid)/processes | jq -r '.resources[] +| select(.type=="web") | .guid')/0 {LOCAL_PATH_TO_FILE} ssh.fr.cloud.gov:tmp/{DESIRED_NAME_OF_FILE} +``` +The items in curly braces are the values that you will manually replace. +These are as follows: +* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt +* DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt + +NOTE: If you'd wish to change what directory these files are uploaded to, you can change `ssh.fr.cloud.gov:tmp/` to `ssh.fr.cloud.gov:{DIRECTORY_YOU_WANT}/`, but be aware that this makes data migration more tricky than it has to be. + +**Get a temp auth code** + +The scp command requires a temporary authentication code. Open a new terminal instance (while keeping the current one open), +and enter the following command: +```shell +cf ssh-code +``` +Copy this code into the password prompt from earlier. + +NOTE: You can use different utilities to copy this onto the clipboard for you. If you are on Windows, try the command `cf ssh-code | clip`. On Mac, this will be `cf ssh-code | pbcopy` + +### STEP 2: Transfer uploaded files to the getgov directory +Due to the nature of how Cloud.gov operates, the getgov directory is dynamically generated whenever the app is built under the tmp/ folder. We can directly upload files to the tmp/ folder but cannot target the generated getgov folder directly, as we need to spin up a shell to access this. From here, we can move those uploaded files into the getgov directory using the `cat` command. Note that you will have to repeat this for each file you want to move, so it is better to use a tar.gz for multiple, and unpack it inside of the `datamigration` folder. + +**SSH into your sandbox** + +```shell +cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} +``` + +**Open a shell** + +```shell +/tmp/lifecycle/shell +``` + +**Move the desired file into the correct directory** + +```shell +cat ../tmp/{filename} > datamigration/{filename} +``` + + +### STEP 3: Load Transition Domain data into TransitionDomain table +Run the following script to transfer the existing data on our .txt files to our DB. +```shell +./manage.py load_transition_domain migrationdata/escrow_domain_contacts.daily.gov.GOV.txt migrationdata/escrow_contacts.daily.gov.GOV.txt migrationdata/escrow_domain_statuses.daily.gov.GOV.txt +``` + +## Load migration data onto our local environments + Transferring this data from these files into our domain tables happens in two steps; ***IMPORTANT: only run the following locally, to avoid publicizing PII in our public repo.*** From e17bb11f4c15666f9cd01122accd8100d264a057 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 23 Oct 2023 12:42:02 -0600 Subject: [PATCH 03/49] Create README.md --- src/migrationdata/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/migrationdata/README.md diff --git a/src/migrationdata/README.md b/src/migrationdata/README.md new file mode 100644 index 000000000..81190ee3f --- /dev/null +++ b/src/migrationdata/README.md @@ -0,0 +1,8 @@ +## Purpose +Use this folder for storing files for the migration process. Should otherwise be empty on local dev environments unless necessary. This folder must exist due to the nature of how data is stored on cloud.gov and the nature of the data we want to send. + +## How do I migrate registrar data? +This process is detailed in [data_migration.md](../../docs/operations/data_migration.md) + +## What kind of files can I store here? +The intent is for PII data or otherwise, but this can exist in any format. Do note that the data contained in this file will be temporary, so after the app is restaged it will lose it. This is ideal for migration files as they write to our DB, but not for something you need to permanently hold onto. \ No newline at end of file From 10841bb97403481f4dff876ef2555890c540ab2f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:51:55 -0600 Subject: [PATCH 04/49] Script for moving files --- docs/operations/data_migration.md | 13 +++- .../commands/cat_files_into_getgov.py | 61 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 src/registrar/management/commands/cat_files_into_getgov.py diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 0a446c7ae..6e8a58b73 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -145,10 +145,21 @@ cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} /tmp/lifecycle/shell ``` +From this directory, run the following command: +```shell +./manage.py cat_files_into_getgov --file_extension txt +``` + +NOTE: This will look for all files in /tmp with the .txt extension, but this can +be changed if you are dealing with different extensions. + +#### Manual method +If the `cat_files_into_getgov.py` script isn't working, follow these steps instead. + **Move the desired file into the correct directory** ```shell -cat ../tmp/{filename} > datamigration/{filename} +cat ../tmp/{filename} > migrationdata/{filename} ``` diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py new file mode 100644 index 000000000..17964f236 --- /dev/null +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -0,0 +1,61 @@ +"""Loads files from /tmp into our sandboxes""" +import glob +import csv +import logging + +import os + +from django.core.management import BaseCommand + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Runs the cat command on files from /tmp into the getgov directory." + + def add_arguments(self, parser): + """Add our two filename arguments.""" + parser.add_argument( + "--file_extension", + default="txt", + help="What file extensions to look for, like txt or gz", + ) + parser.add_argument("--directory", default="migrationdata", help="Desired directory") + + def handle(self, **options): + file_extension: str = options.get("file_extension").lstrip('.') + directory = options.get("directory") + + # file_extension is always coerced as str, Truthy is OK to use here. + if not file_extension or not isinstance(file_extension, str): + raise ValueError(f"Invalid file extension '{file_extension}'") + + matching_extensions = glob.glob(f'../tmp/*.{file_extension}') + if not matching_extensions: + logger.error(f"No files with the extension {file_extension} found") + + for src_file_path in matching_extensions: + filename = os.path.basename(src_file_path) + do_command = True + exit_status: int + + desired_file_path = f'{directory}/{filename}' + if os.path.exists(desired_file_path): + replace = input(f'{desired_file_path} already exists. Do you want to replace it? (y/n) ') + if replace.lower() != 'y': + do_command = False + + if do_command: + copy_from = f"../tmp/{filename}" + self.cat(copy_from, desired_file_path) + exit_status = os.system(f'cat ../tmp/{filename} > {desired_file_path}') + + if exit_status == 0: + logger.info(f"Successfully copied {filename}") + else: + logger.info(f"Failed to copy {filename}") + + def cat(self, copy_from, copy_to): + exit_status = os.system(f'cat {copy_from} > {copy_to}') + return exit_status \ No newline at end of file From c713b8958f0ce0585f969d0530b080a4e0bab609 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 23 Oct 2023 14:54:02 -0600 Subject: [PATCH 05/49] added test_domain_migration script and terminal_helper. Still needs work Signed-off-by: CocoByte --- .../commands/test_domain_migration.py | 223 +++++++++++++++--- .../management/commands/utility/enums.py | 0 .../commands/utility/terminal_helper.py | 50 ++++ 3 files changed, 235 insertions(+), 38 deletions(-) delete mode 100644 src/registrar/management/commands/utility/enums.py create mode 100644 src/registrar/management/commands/utility/terminal_helper.py diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index 7bdda7643..19631140a 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -9,59 +9,76 @@ from django.core.management import BaseCommand from registrar.models import TransitionDomain from registrar.models import Domain from registrar.models import DomainInvitation +from registrar.models.domain_information import DomainInformation + +from registrar.management.commands.utility.terminal_helper import TerminalColors +from registrar.management.commands.utility.terminal_helper import TerminalHelper + +from registrar.management.commands.load_transition_domain import Command as load_transition_domain_command logger = logging.getLogger(__name__) - -class termColors: - """Colors for terminal outputs - (makes reading the logs WAY easier)""" - - HEADER = "\033[95m" - OKBLUE = "\033[94m" - OKCYAN = "\033[96m" - OKGREEN = "\033[92m" - YELLOW = "\033[93m" - FAIL = "\033[91m" - ENDC = "\033[0m" - BOLD = "\033[1m" - UNDERLINE = "\033[4m" - BackgroundLightYellow = "\033[103m" - - class Command(BaseCommand): help = """ """ def add_arguments(self, parser): + """ + OPTIONAL ARGUMENTS: + --debug + A boolean (default to true), which activates additional print statements + """ + + parser.add_argument("--runLoaders", + help="Runs all scripts (in sequence) for transition domain migrations", + action=argparse.BooleanOptionalAction) + + # The file arguments have default values for running in the sandbox + parser.add_argument( + "--loaderDirectory", + default="migrationData/", + help="The location of the files used for load_transition_domain migration script" + ) + parser.add_argument( + "domain_contacts_filename", + default="escrow_domain_contacts.daily.gov.GOV.txt", + help="Data file with domain contact information" + ) + parser.add_argument( + "contacts_filename", + default="escrow_contacts.daily.gov.GOV.txt", + help="Data file with contact information", + ) + parser.add_argument( + "domain_statuses_filename", + default="escrow_domain_statuses.daily.gov.GOV.txt", + help="Data file with domain status information" + ) + + parser.add_argument("--sep", default="|", help="Delimiter character") + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) parser.add_argument( - "--limitParse", - default=0, - help="Sets max number of entries to load, set to 0 to load all entries", + "--limitParse", default=0, help="Sets max number of entries to load" + ) + + parser.add_argument( + "--resetTable", + help="Deletes all data in the TransitionDomain table", + action=argparse.BooleanOptionalAction, ) def print_debug_mode_statements( - self, debug_on: bool, debug_max_entries_to_parse: int + self, debug_on: bool ): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" self.print_debug( debug_on, - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. - {termColors.ENDC} - """, - ) - self.print_debug( - debug_max_entries_to_parse > 0, - f"""{termColors.OKCYAN} - ----------LIMITER ON---------- - Parsing of entries will be limited to - {debug_max_entries_to_parse} lines per file.") - Detailed print statements activated. - {termColors.ENDC} + {TerminalColors.ENDC} """, ) @@ -74,21 +91,133 @@ class Command(BaseCommand): if print_condition: logger.info(print_statement) + def compare_tables(self, debug_on): + logger.info( + f"""{TerminalColors.OKCYAN} + ============= BEGINNING ANALYSIS =============== + {TerminalColors.ENDC} + """ + ) + + #TODO: would filteredRelation be faster? + for transition_domain in TransitionDomain.objects.all():# DEBUG: + transition_domain_name = transition_domain.domain_name + transition_domain_email = transition_domain.username + + self.print_debug( + debug_on, + f"{TerminalColors.OKCYAN}Checking: {transition_domain_name} {TerminalColors.ENDC}", # noqa + ) + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] + + # Check Domain table + matching_domains = Domain.objects.filter(name=transition_domain_name) + # Check Domain Information table + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + # Check Domain Invitation table + matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), + domain__name=transition_domain_name) + + if len(matching_domains) == 0: + missing_domains.append(transition_domain_name) + elif len(matching_domains) > 1: + duplicate_domains.append(transition_domain_name) + if len(matching_domain_informations) == 0: + missing_domain_informations.append(transition_domain_name) + if len(matching_domain_invitations) == 0: + missing_domain_invites.append(transition_domain_name) + + total_missing_domains = len(missing_domains) + total_duplicate_domains = len(duplicate_domains) + total_missing_domain_informations = len(missing_domain_informations) + total_missing_domain_invitations = len(missing_domain_invites) + + missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) + duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) + missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) + missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) + + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED ANALYSIS =============== + + {total_missing_domains} Missing Domains: + (These are transition domains that are missing from the Domain Table) + {TerminalColors.YELLOW}{missing_domains_as_string}{TerminalColors.OKGREEN} + + {total_duplicate_domains} Duplicate Domains: + (These are transition domains which have duplicate entries in the Domain Table) + {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_informations} Domains Information Entries missing: + (These are transition domains which have no entries in the Domain Information Table) + {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_invitations} Domain Invitations missing: + (These are transition domains which have no entires in the Domain Invitation Table) + {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} + {TerminalColors.ENDC} + """ + ) + def run_migration_scripts(self, + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename): + + files_are_correct = TerminalHelper.query_yes_no( + f""" + {TerminalColors.YELLOW} + PLEASE CHECK: + The loader scripts expect to find the following files: + - domain contacts: {domain_contacts_filename} + - contacts: {contacts_filename} + - domain statuses: {domain_statuses_filename} + + The files should be at the following directory; + {file_location} + + Does this look correct?{TerminalColors.ENDC}""" + ) + + if not files_are_correct: + # prompt the user to provide correct file inputs + logger.info(f""" + {TerminalColors.YELLOW} + PLEASE Re-Run the script with the correct file location and filenames: + EXAMPLE: + + + """) + return + load_transition_domain_command.handle( + domain_contacts_filename, + contacts_filename, + domain_statuses_filename + ) + def handle( self, + # domain_contacts_filename, + # contacts_filename, + # domain_statuses_filename, **options, ): """ - Do a diff between the transition_domain and the following tables: + Does a diff between the transition_domain and the following tables: domain, domain_information and the domain_invitation. - It should: - - Print any domains that exist in the transition_domain table + Produces the following report (printed to the terminal): + #1 - Print any domains that exist in the transition_domain table but not in their corresponding domain, domain information or domain invitation tables. - - Print which table this domain is missing from - - Check for duplicate entries in domain or + #2 - Print which table this domain is missing from + #3- Check for duplicate entries in domain or domain_information tables and print which are duplicates and in which tables @@ -105,3 +234,21 @@ class Command(BaseCommand): ** Triggers table reset ** """ + # Get --debug argument + debug_on = options.get("debug") + # Get --runLoaders argument + run_loaders_on = options.get("runLoaders") + + # Analyze tables for corrupt data... + self.compare_tables(debug_on) + + # Run migration scripts if specified by user... + if run_loaders_on: + file_location = options.get("loaderDirectory") + # domain_contacts_filename = options.get("domain_contacts_filename") + # contacts_filename = options.get("contacts_filename") + # domain_statuses_filename = options.get("domain_statuses_filename") + # self.run_migration_scripts(file_location, + # domain_contacts_filename, + # contacts_filename, + # domain_statuses_filename) \ No newline at end of file diff --git a/src/registrar/management/commands/utility/enums.py b/src/registrar/management/commands/utility/enums.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py new file mode 100644 index 000000000..da2a3a54a --- /dev/null +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -0,0 +1,50 @@ +import logging + +logger = logging.getLogger(__name__) + +class TerminalColors: + """Colors for terminal outputs + (makes reading the logs WAY easier)""" + + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKCYAN = "\033[96m" + OKGREEN = "\033[92m" + YELLOW = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + BackgroundLightYellow = "\033[103m" + +class TerminalHelper: + + def query_yes_no(question: str, default="yes") -> bool: + """Ask a yes/no question via raw_input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + """ + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + logger.info(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return valid[default] + elif choice in valid: + return valid[choice] + else: + logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") \ No newline at end of file From d5c0ac7a0c7cda5a28b8ee20aa3782f3d4912134 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 23 Oct 2023 22:22:54 -0600 Subject: [PATCH 06/49] attempt 1 for executing external functions -- note, this doesn't work Signed-off-by: CocoByte --- .../commands/load_transition_domain.py | 50 +++++--- .../commands/test_domain_migration.py | 117 ++++++++++++------ 2 files changed, 114 insertions(+), 53 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 206589c33..1212870b5 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -297,29 +297,20 @@ class Command(BaseCommand): ) TransitionDomain.objects.all().delete() - def handle( # noqa: C901 - self, + def parse_files(self, # noqa: C901 domain_contacts_filename, contacts_filename, domain_statuses_filename, - **options, - ): - """Parse the data files and create TransitionDomains.""" - sep = options.get("sep") + reset_table, + sep, + debug_on, + debug_max_entries_to_parse): # If --resetTable was used, prompt user to confirm # deletion of table data - if options.get("resetTable"): + if reset_table: self.prompt_table_reset() - # Get --debug argument - debug_on = options.get("debug") - - # Get --LimitParse argument - debug_max_entries_to_parse = int( - options.get("limitParse") - ) # set to 0 to parse all entries - # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) @@ -522,3 +513,32 @@ class Command(BaseCommand): duplicate_domain_user_combos, duplicate_domains, users_without_email ) self.print_summary_status_findings(domains_without_status, outlier_statuses) + + def handle( + self, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + **options, + ): + """Parse the data files and create TransitionDomains.""" + # Get --sep argument + sep = options.get("sep") + + # Get --resetTable argument + reset_table = options.get("resetTable") + + # Get --debug argument + debug_on = options.get("debug") + + # Get --limitParse argument + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + self.parse_files(domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse) diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index 19631140a..f5154e0fe 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -35,26 +35,39 @@ class Command(BaseCommand): # The file arguments have default values for running in the sandbox parser.add_argument( "--loaderDirectory", - default="migrationData/", + default="migrationData", help="The location of the files used for load_transition_domain migration script" ) parser.add_argument( - "domain_contacts_filename", - default="escrow_domain_contacts.daily.gov.GOV.txt", - help="Data file with domain contact information" - ) - parser.add_argument( - "contacts_filename", - default="escrow_contacts.daily.gov.GOV.txt", - help="Data file with contact information", - ) - parser.add_argument( - "domain_statuses_filename", - default="escrow_domain_statuses.daily.gov.GOV.txt", - help="Data file with domain status information" + "--loaderFilenames", + default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information""" ) - parser.add_argument("--sep", default="|", help="Delimiter character") + # parser.add_argument( + # "domain_contacts_filename", + # default="escrow_domain_contacts.daily.gov.GOV.txt", + # help="Data file with domain contact information" + # ) + # parser.add_argument( + # "contacts_filename", + # default="escrow_contacts.daily.gov.GOV.txt", + # help="Data file with contact information", + # ) + # parser.add_argument( + # "domain_statuses_filename", + # default="escrow_domain_statuses.daily.gov.GOV.txt", + # help="Data file with domain status information" + # ) + + parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -165,23 +178,40 @@ class Command(BaseCommand): ) def run_migration_scripts(self, - file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename): + options): + + file_location = options.get("loaderDirectory")+"/" + filenames = options.get("loaderFilenames").split() + if len(filenames) < 3: + filenames_as_string = "{}".format(", ".join(map(str, filenames))) + logger.info(f""" + {TerminalColors.FAIL} + --loaderFilenames expected 3 filenames to follow it, + but only {len(filenames)} were given: + {filenames_as_string} + + PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN + ============= TERMINATING ============= + {TerminalColors.ENDC} + """) + return + domain_contacts_filename = filenames[0] + contacts_filename = filenames[1] + domain_statuses_filename = filenames[2] files_are_correct = TerminalHelper.query_yes_no( f""" {TerminalColors.YELLOW} - PLEASE CHECK: - The loader scripts expect to find the following files: + *** IMPORTANT: VERIFY THE FOLLOWING *** + + The migration scripts are looking in directory.... + {file_location} + + ....for the following files: - domain contacts: {domain_contacts_filename} - contacts: {contacts_filename} - domain statuses: {domain_statuses_filename} - The files should be at the following directory; - {file_location} - Does this look correct?{TerminalColors.ENDC}""" ) @@ -190,22 +220,37 @@ class Command(BaseCommand): logger.info(f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: - EXAMPLE: + EXAMPLE: + docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt """) return - load_transition_domain_command.handle( - domain_contacts_filename, - contacts_filename, - domain_statuses_filename - ) + + # Get --sep argument + sep = options.get("sep") + + # Get --resetTable argument + reset_table = options.get("resetTable") + + # Get --debug argument + debug_on = options.get("debug") + + # Get --limitParse argument + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + load_transition_domain_command.parse_files(load_transition_domain_command, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse) def handle( self, - # domain_contacts_filename, - # contacts_filename, - # domain_statuses_filename, **options, ): """ @@ -244,11 +289,7 @@ class Command(BaseCommand): # Run migration scripts if specified by user... if run_loaders_on: - file_location = options.get("loaderDirectory") # domain_contacts_filename = options.get("domain_contacts_filename") # contacts_filename = options.get("contacts_filename") # domain_statuses_filename = options.get("domain_statuses_filename") - # self.run_migration_scripts(file_location, - # domain_contacts_filename, - # contacts_filename, - # domain_statuses_filename) \ No newline at end of file + self.run_migration_scripts(options) \ No newline at end of file From cb3cfe3a6d5c34bf0c2ba149bacec196eda3620f Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 00:23:45 -0600 Subject: [PATCH 07/49] finished options for running loader scripts. Added boilerplate for simulating logins --- .../commands/load_transition_domain.py | 50 ++---- .../commands/test_domain_migration.py | 156 +++++++++++++++--- .../commands/utility/terminal_helper.py | 11 +- 3 files changed, 162 insertions(+), 55 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 1212870b5..206589c33 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -297,20 +297,29 @@ class Command(BaseCommand): ) TransitionDomain.objects.all().delete() - def parse_files(self, # noqa: C901 + def handle( # noqa: C901 + self, domain_contacts_filename, contacts_filename, domain_statuses_filename, - reset_table, - sep, - debug_on, - debug_max_entries_to_parse): + **options, + ): + """Parse the data files and create TransitionDomains.""" + sep = options.get("sep") # If --resetTable was used, prompt user to confirm # deletion of table data - if reset_table: + if options.get("resetTable"): self.prompt_table_reset() + # Get --debug argument + debug_on = options.get("debug") + + # Get --LimitParse argument + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # set to 0 to parse all entries + # print message to terminal about which args are in use self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) @@ -513,32 +522,3 @@ class Command(BaseCommand): duplicate_domain_user_combos, duplicate_domains, users_without_email ) self.print_summary_status_findings(domains_without_status, outlier_statuses) - - def handle( - self, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - **options, - ): - """Parse the data files and create TransitionDomains.""" - # Get --sep argument - sep = options.get("sep") - - # Get --resetTable argument - reset_table = options.get("resetTable") - - # Get --debug argument - debug_on = options.get("debug") - - # Get --limitParse argument - debug_max_entries_to_parse = int( - options.get("limitParse") - ) # set to 0 to parse all entries - self.parse_files(domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - debug_max_entries_to_parse) diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index f5154e0fe..bc9efa9df 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -1,10 +1,13 @@ import logging import argparse -import sys +import os + +from django.test import Client from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand +from django.contrib.auth import get_user_model from registrar.models import TransitionDomain from registrar.models import Domain @@ -31,8 +34,12 @@ class Command(BaseCommand): parser.add_argument("--runLoaders", help="Runs all scripts (in sequence) for transition domain migrations", action=argparse.BooleanOptionalAction) + + parser.add_argument("--triggerLogins", + help="Simulates a user login for each user in domain invitation", + action=argparse.BooleanOptionalAction) - # The file arguments have default values for running in the sandbox + # The following file arguments have default values for running in the sandbox parser.add_argument( "--loaderDirectory", default="migrationData", @@ -177,9 +184,79 @@ class Command(BaseCommand): """ ) + def run_load_transition_domain_script(self, + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse): + load_transition_domain_command_string = "./manage.py load_transition_domain " + load_transition_domain_command_string += file_location+domain_contacts_filename + " " + load_transition_domain_command_string += file_location+contacts_filename + " " + load_transition_domain_command_string += file_location+domain_statuses_filename + " " + + if sep is not None and sep != "|": + load_transition_domain_command_string += f"--sep {sep} " + + if reset_table: + load_transition_domain_command_string += "--resetTable " + + if debug_on: + load_transition_domain_command_string += "--debug " + + if debug_max_entries_to_parse > 0: + load_transition_domain_command_string += f"--limitParse {debug_max_entries_to_parse} " + + proceed_load_transition_domain = TerminalHelper.query_yes_no( + f"""{TerminalColors.OKCYAN} + ===================================== + Running load_transition_domain script + ===================================== + + {load_transition_domain_command_string} + {TerminalColors.FAIL} + Proceed? + {TerminalColors.ENDC}""" + ) + + if not proceed_load_transition_domain: + return + logger.info(f"""{TerminalColors.OKCYAN} + ==== EXECUTING... ==== + {TerminalColors.ENDC}""") + os.system(f"{load_transition_domain_command_string}") + + def run_transfer_script(self, debug_on): + command_string = "./manage.py transfer_transition_domains_to_domains " + + if debug_on: + command_string += "--debug " + + + proceed_load_transition_domain = TerminalHelper.query_yes_no( + f"""{TerminalColors.OKCYAN} + ===================================================== + Running transfer_transition_domains_to_domains script + ===================================================== + + {command_string} + {TerminalColors.FAIL} + Proceed? + {TerminalColors.ENDC}""" + ) + + if not proceed_load_transition_domain: + return + logger.info(f"""{TerminalColors.OKCYAN} + ==== EXECUTING... ==== + {TerminalColors.ENDC}""") + os.system(f"{command_string}") + def run_migration_scripts(self, options): - file_location = options.get("loaderDirectory")+"/" filenames = options.get("loaderFilenames").split() if len(filenames) < 3: @@ -201,7 +278,7 @@ class Command(BaseCommand): files_are_correct = TerminalHelper.query_yes_no( f""" - {TerminalColors.YELLOW} + {TerminalColors.OKCYAN} *** IMPORTANT: VERIFY THE FOLLOWING *** The migration scripts are looking in directory.... @@ -210,8 +287,9 @@ class Command(BaseCommand): ....for the following files: - domain contacts: {domain_contacts_filename} - contacts: {contacts_filename} - - domain statuses: {domain_statuses_filename} + - domain statuses: {domain_statuses_filename}y + {TerminalColors.FAIL} Does this look correct?{TerminalColors.ENDC}""" ) @@ -240,14 +318,38 @@ class Command(BaseCommand): debug_max_entries_to_parse = int( options.get("limitParse") ) # set to 0 to parse all entries - load_transition_domain_command.parse_files(load_transition_domain_command, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - debug_max_entries_to_parse) + + self.run_load_transition_domain_script(file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + debug_max_entries_to_parse) + + self.run_transfer_script(debug_on) + + def simulate_user_logins(self, debug_on): + logger.info(f"""{TerminalColors.OKCYAN} + ================== + SIMULATING LOGINS + ================== + {TerminalColors.ENDC} + """) + for invite in DomainInvitation.objects.all(): + #DEBUG: + TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") + # get a user with this email address + User = get_user_model() + try: + user = User.objects.get(email=invite.email) + #DEBUG: + TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") + Client.force_login(user) + except User.DoesNotExist: + #TODO: how should we handle this? + logger.warn(f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""") def handle( self, @@ -283,13 +385,29 @@ class Command(BaseCommand): debug_on = options.get("debug") # Get --runLoaders argument run_loaders_on = options.get("runLoaders") + # Get --triggerLogins argument + simulate_user_login_enabled = options.get("triggerLogins") - # Analyze tables for corrupt data... - self.compare_tables(debug_on) + prompt_continuation_of_analysis = False # Run migration scripts if specified by user... if run_loaders_on: - # domain_contacts_filename = options.get("domain_contacts_filename") - # contacts_filename = options.get("contacts_filename") - # domain_statuses_filename = options.get("domain_statuses_filename") - self.run_migration_scripts(options) \ No newline at end of file + self.run_migration_scripts(options) + prompt_continuation_of_analysis = True + + # Simulate user login for each user in domain invitation if sepcified by user + if simulate_user_login_enabled: + self.simulate_user_logins(debug_on) + prompt_continuation_of_analysis = True + + analyze_tables = True + if prompt_continuation_of_analysis: + analyze_tables = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with table analysis? + {TerminalColors.ENDC}""" + ) + + # Analyze tables for corrupt data... + if analyze_tables: + self.compare_tables(debug_on) \ No newline at end of file diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index da2a3a54a..ec7580e21 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -47,4 +47,13 @@ class TerminalHelper: elif choice in valid: return valid[choice] else: - logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") \ No newline at end of file + logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + + def print_debug(print_condition: bool, print_statement: str): + """This function reduces complexity of debug statements + in other functions. + It uses the logger to write the given print_statement to the + terminal if print_condition is TRUE""" + # DEBUG: + if print_condition: + logger.info(print_statement) \ No newline at end of file From 852f22a57a6148c1fbde15b2fb43e6a57e84b71b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 10:16:14 -0600 Subject: [PATCH 08/49] revising login simulation -- committing for sandbox testing --- docs/operations/data_migration.md | 5 +++ .../commands/test_domain_migration.py | 43 +++++++++++++------ .../commands/utility/terminal_helper.py | 2 +- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 6e8a58b73..3e22a64f6 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -84,6 +84,10 @@ FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain na FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses +We need to run a few scripts to parse these files into our domain tables. +We can do this both locally and in a sandbox. + +## OPTION 1: SANDBOX ## Load migration data onto a production or sandbox environment **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. Do not use this method to store data you want to keep around permanently. @@ -169,6 +173,7 @@ Run the following script to transfer the existing data on our .txt files to our ./manage.py load_transition_domain migrationdata/escrow_domain_contacts.daily.gov.GOV.txt migrationdata/escrow_contacts.daily.gov.GOV.txt migrationdata/escrow_domain_statuses.daily.gov.GOV.txt ``` +## OPTION 2: LOCAL ## Load migration data onto our local environments Transferring this data from these files into our domain tables happens in two steps; diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index bc9efa9df..12f4bd745 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -12,7 +12,8 @@ from django.contrib.auth import get_user_model from registrar.models import TransitionDomain from registrar.models import Domain from registrar.models import DomainInvitation -from registrar.models.domain_information import DomainInformation +from registrar.models import DomainInformation +from registrar.models import User from registrar.management.commands.utility.terminal_helper import TerminalColors from registrar.management.commands.utility.terminal_helper import TerminalHelper @@ -337,19 +338,33 @@ class Command(BaseCommand): ================== {TerminalColors.ENDC} """) - for invite in DomainInvitation.objects.all(): - #DEBUG: - TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") - # get a user with this email address - User = get_user_model() - try: - user = User.objects.get(email=invite.email) - #DEBUG: - TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") - Client.force_login(user) - except User.DoesNotExist: - #TODO: how should we handle this? - logger.warn(f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""") + # for invite in DomainInvitation.objects.all(): + # #DEBUG: + # TerminalHelper.print_conditional(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") + # # get a user with this email address + # user_exists = User.objects.filter(email=invite.email).exists() + # user, _ = User.objects.get_or_create(email=invite.email) + # #DEBUG: + # TerminalHelper.print_conditional(user_exists,f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") + # TerminalHelper.print_conditional(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") + # user.first_login() + # if not user_exists: + # logger.warn(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + # user.delete() + + # for invite in DomainInvitation.objects.all(): + # #DEBUG: + # TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") + # # get a user with this email address + # User = get_user_model() + # try: + # user = User.objects.get(email=invite.email) + # #DEBUG: + # TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") + # Client.force_login(user) + # except User.DoesNotExist: + # #TODO: how should we handle this? + # logger.warn(f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""") def handle( self, diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index ec7580e21..abfcdcae0 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -49,7 +49,7 @@ class TerminalHelper: else: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") - def print_debug(print_condition: bool, print_statement: str): + def print_conditional(print_condition: bool, print_statement: str): """This function reduces complexity of debug statements in other functions. It uses the logger to write the given print_statement to the From 4f00865ff293e02e0db25d193b286cf1a598d785 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:51:36 -0600 Subject: [PATCH 09/49] Update cat_files_into_getgov.py --- src/registrar/management/commands/cat_files_into_getgov.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 17964f236..9111263a6 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -54,8 +54,10 @@ class Command(BaseCommand): if exit_status == 0: logger.info(f"Successfully copied {filename}") else: - logger.info(f"Failed to copy {filename}") + logger.error(f"Failed to copy {filename}") def cat(self, copy_from, copy_to): + """Runs the cat command to + copy_from a location to copy_to a location""" exit_status = os.system(f'cat {copy_from} > {copy_to}') return exit_status \ No newline at end of file From 4f6b5e2b06cbfda2611c58f9e660cfbad79c5d22 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 15:13:56 -0600 Subject: [PATCH 10/49] added comments and typehints. Refactored a little. --- .../commands/load_transition_domain.py | 124 ++--- .../commands/test_domain_migration.py | 463 +++++++++++------- .../transfer_transition_domains_to_domains.py | 111 ++--- .../commands/utility/terminal_helper.py | 1 + 4 files changed, 371 insertions(+), 328 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 206589c33..ee1e8c4ea 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -9,56 +9,15 @@ from django.core.management import BaseCommand from registrar.models import TransitionDomain + +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper +) + logger = logging.getLogger(__name__) -class termColors: - """Colors for terminal outputs - (makes reading the logs WAY easier)""" - - HEADER = "\033[95m" - OKBLUE = "\033[94m" - OKCYAN = "\033[96m" - OKGREEN = "\033[92m" - YELLOW = "\033[93m" - FAIL = "\033[91m" - ENDC = "\033[0m" - BOLD = "\033[1m" - UNDERLINE = "\033[4m" - BackgroundLightYellow = "\033[103m" - - -def query_yes_no(question: str, default="yes") -> bool: - """Ask a yes/no question via raw_input() and return their answer. - - "question" is a string that is presented to the user. - "default" is the presumed answer if the user just hits . - It must be "yes" (the default), "no" or None (meaning - an answer is required of the user). - - The "answer" return value is True for "yes" or False for "no". - """ - valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} - if default is None: - prompt = " [y/n] " - elif default == "yes": - prompt = " [Y/n] " - elif default == "no": - prompt = " [y/N] " - else: - raise ValueError("invalid default answer: '%s'" % default) - - while True: - logger.info(question + prompt) - choice = input().lower() - if default is not None and choice == "": - return valid[default] - elif choice in valid: - return valid[choice] - else: - logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") - - class Command(BaseCommand): help = """Loads data for domains that are in transition (populates transition_domain model objects).""" @@ -110,20 +69,20 @@ class Command(BaseCommand): or --limitParse are in use""" if debug_on: logger.info( - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. - {termColors.ENDC} + {TerminalColors.ENDC} """ ) if debug_max_entries_to_parse > 0: logger.info( - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} ----------LIMITER ON---------- Parsing of entries will be limited to {debug_max_entries_to_parse} lines per file.") Detailed print statements activated. - {termColors.ENDC} + {TerminalColors.ENDC} """ ) @@ -195,7 +154,7 @@ class Command(BaseCommand): ", ".join(map(str, duplicate_domain_user_combos)) ) logger.warning( - f"{termColors.YELLOW} No e-mails found for users: {users_without_email_as_string}" # noqa + f"{TerminalColors.YELLOW} No e-mails found for users: {users_without_email_as_string}" # noqa ) if total_duplicate_pairs > 0 or total_duplicate_domains > 0: duplicate_pairs_as_string = "{}".format( @@ -205,7 +164,7 @@ class Command(BaseCommand): ", ".join(map(str, duplicate_domains)) ) logger.warning( - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} ----DUPLICATES FOUND----- @@ -218,7 +177,7 @@ class Command(BaseCommand): the supplied data files; {duplicate_domains_as_string} - {termColors.ENDC}""" + {TerminalColors.ENDC}""" ) def print_summary_status_findings( @@ -237,7 +196,7 @@ class Command(BaseCommand): ", ".join(map(str, domains_without_status)) ) logger.warning( - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} -------------------------------------------- Found {total_domains_without_status} domains @@ -245,7 +204,7 @@ class Command(BaseCommand): --------------------------------------------- {domains_without_status_as_string} - {termColors.ENDC}""" + {TerminalColors.ENDC}""" ) if total_outlier_statuses > 0: @@ -253,7 +212,7 @@ class Command(BaseCommand): ", ".join(map(str, outlier_statuses)) ) # noqa logger.warning( - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} -------------------------------------------- Found {total_outlier_statuses} unaccounted @@ -264,36 +223,27 @@ class Command(BaseCommand): (defaulted to Ready): {domains_without_status_as_string} - {termColors.ENDC}""" + {TerminalColors.ENDC}""" ) - def print_debug(self, print_condition: bool, print_statement: str): - """This function reduces complexity of debug statements - in other functions. - It uses the logger to write the given print_statement to the - terminal if print_condition is TRUE""" - # DEBUG: - if print_condition: - logger.info(print_statement) - def prompt_table_reset(self): """Brings up a prompt in the terminal asking if the user wishes to delete data in the TransitionDomain table. If the user confirms, deletes all the data in the TransitionDomain table""" - confirm_reset = query_yes_no( + confirm_reset = TerminalHelper.query_yes_no( f""" - {termColors.FAIL} + {TerminalColors.FAIL} WARNING: Resetting the table will permanently delete all the data! - Are you sure you want to continue?{termColors.ENDC}""" + Are you sure you want to continue?{TerminalColors.ENDC}""" ) if confirm_reset: logger.info( - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} ----------Clearing Table Data---------- (please wait) - {termColors.ENDC}""" + {TerminalColors.ENDC}""" ) TransitionDomain.objects.all().delete() @@ -428,18 +378,18 @@ class Command(BaseCommand): ) if existing_domain is not None: # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"{termColors.YELLOW} DUPLICATE file entries found for domain: {new_entry_domain_name} {termColors.ENDC}", # noqa + f"{TerminalColors.YELLOW} DUPLICATE file entries found for domain: {new_entry_domain_name} {TerminalColors.ENDC}", # noqa ) if new_entry_domain_name not in duplicate_domains: duplicate_domains.append(new_entry_domain_name) if existing_domain_user_pair is not None: # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.YELLOW} DUPLICATE file entries found for domain - user {termColors.BackgroundLightYellow} PAIR {termColors.ENDC}{termColors.YELLOW}: - {new_entry_domain_name} - {new_entry_email} {termColors.ENDC}""", # noqa + f"""{TerminalColors.YELLOW} DUPLICATE file entries found for domain - user {TerminalColors.BackgroundLightYellow} PAIR {TerminalColors.ENDC}{TerminalColors.YELLOW}: + {new_entry_domain_name} - {new_entry_email} {TerminalColors.ENDC}""", # noqa ) if existing_domain_user_pair not in duplicate_domain_user_combos: duplicate_domain_user_combos.append(existing_domain_user_pair) @@ -456,20 +406,20 @@ class Command(BaseCommand): if existing_entry.status != new_entry_status: # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"{termColors.OKCYAN}" + f"{TerminalColors.OKCYAN}" f"Updating entry: {existing_entry}" f"Status: {existing_entry.status} > {new_entry_status}" # noqa f"Email Sent: {existing_entry.email_sent} > {new_entry_emailSent}" # noqa - f"{termColors.ENDC}", + f"{TerminalColors.ENDC}", ) existing_entry.status = new_entry_status existing_entry.email_sent = new_entry_emailSent existing_entry.save() except TransitionDomain.MultipleObjectsReturned: logger.info( - f"{termColors.FAIL}" + f"{TerminalColors.FAIL}" f"!!! ERROR: duplicate entries exist in the" f"transtion_domain table for domain:" f"{new_entry_domain_name}" @@ -488,9 +438,9 @@ class Command(BaseCommand): total_new_entries += 1 # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"{termColors.OKCYAN} Adding entry {total_new_entries}: {new_entry} {termColors.ENDC}", # noqa + f"{TerminalColors.OKCYAN} Adding entry {total_new_entries}: {new_entry} {TerminalColors.ENDC}", # noqa ) # Check Parse limit and exit loop if needed @@ -499,20 +449,20 @@ class Command(BaseCommand): and debug_max_entries_to_parse != 0 ): logger.info( - f"{termColors.YELLOW}" + f"{TerminalColors.YELLOW}" f"----PARSE LIMIT REACHED. HALTING PARSER.----" - f"{termColors.ENDC}" + f"{TerminalColors.ENDC}" ) break TransitionDomain.objects.bulk_create(to_create) logger.info( - f"""{termColors.OKGREEN} + f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Created {total_new_entries} transition domain entries, updated {total_updated_domain_entries} transition domain entries - {termColors.ENDC} + {TerminalColors.ENDC} """ ) diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index 12f4bd745..d71f88c36 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -1,5 +1,13 @@ +"""Data migration: + 1 - generates a report of data integrity across all + transition domain related tables + 2 - allows users to run all migration scripts for + transition domain data +""" + import logging import argparse +import sys import os from django.test import Client @@ -7,18 +15,19 @@ from django.test import Client from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand -from django.contrib.auth import get_user_model -from registrar.models import TransitionDomain -from registrar.models import Domain -from registrar.models import DomainInvitation -from registrar.models import DomainInformation -from registrar.models import User +from registrar.models import ( + Domain, + DomainInformation, + DomainInvitation, + TransitionDomain, + User, +) -from registrar.management.commands.utility.terminal_helper import TerminalColors -from registrar.management.commands.utility.terminal_helper import TerminalHelper - -from registrar.management.commands.load_transition_domain import Command as load_transition_domain_command +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper +) logger = logging.getLogger(__name__) @@ -28,8 +37,45 @@ class Command(BaseCommand): def add_arguments(self, parser): """ OPTIONAL ARGUMENTS: + --runLoaders + A boolean (default to true), which triggers running + all scripts (in sequence) for transition domain migrations + + --triggerLogins + A boolean (default to true), which triggers running + simulations of user logins for each user in domain invitation + + --loaderDirectory + The location of the files used for load_transition_domain migration script + EXAMPLE USAGE: + > --loaderDirectory /app/tmp + + --loaderFilenames + The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + EXAMPLE USAGE: + > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information + + --sep + Delimiter for the loaders to correctly parse the given text files. + (usually this can remain at default value of |) + --debug A boolean (default to true), which activates additional print statements + + --limitParse + Used by the loaders (load_transition_domain) to set the limit for the + number of data entries to insert. Set to 0 (or just don't use this + argument) to parse every entry. This was provided primarily for testing + purposes + + --resetTable + Used by the loaders to trigger a prompt for deleting all table entries. + Useful for testing purposes, but USE WITH CAUTION """ parser.add_argument("--runLoaders", @@ -59,22 +105,6 @@ class Command(BaseCommand): - domain_statuses_filename is the Data file with domain status information""" ) - # parser.add_argument( - # "domain_contacts_filename", - # default="escrow_domain_contacts.daily.gov.GOV.txt", - # help="Data file with domain contact information" - # ) - # parser.add_argument( - # "contacts_filename", - # default="escrow_contacts.daily.gov.GOV.txt", - # help="Data file with contact information", - # ) - # parser.add_argument( - # "domain_statuses_filename", - # default="escrow_domain_statuses.daily.gov.GOV.txt", - # help="Data file with domain status information" - # ) - parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -89,30 +119,22 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - def print_debug_mode_statements( - self, debug_on: bool - ): - """Prints additional terminal statements to indicate if --debug - or --limitParse are in use""" - self.print_debug( - debug_on, - f"""{TerminalColors.OKCYAN} - ----------DEBUG MODE ON---------- - Detailed print statements activated. - {TerminalColors.ENDC} - """, - ) + - def print_debug(self, print_condition: bool, print_statement: str): - """This function reduces complexity of debug statements - in other functions. - It uses the logger to write the given print_statement to the - terminal if print_condition is TRUE""" - # DEBUG: - if print_condition: - logger.info(print_statement) - - def compare_tables(self, debug_on): + def compare_tables(self, debug_on: bool): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + + Produces the following report (printed to the terminal): + #1 - Print any domains that exist in the transition_domain table + but not in their corresponding domain, domain information or + domain invitation tables. + #2 - Print which table this domain is missing from + #3- Check for duplicate entries in domain or + domain_information tables and print which are + duplicates and in which tables + """ + logger.info( f"""{TerminalColors.OKCYAN} ============= BEGINNING ANALYSIS =============== @@ -121,20 +143,20 @@ class Command(BaseCommand): ) #TODO: would filteredRelation be faster? + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] for transition_domain in TransitionDomain.objects.all():# DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username - self.print_debug( + TerminalHelper.print_conditional( debug_on, f"{TerminalColors.OKCYAN}Checking: {transition_domain_name} {TerminalColors.ENDC}", # noqa ) - missing_domains = [] - duplicate_domains = [] - missing_domain_informations = [] - missing_domain_invites = [] - # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table @@ -144,12 +166,16 @@ class Command(BaseCommand): domain__name=transition_domain_name) if len(matching_domains) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") missing_domains.append(transition_domain_name) elif len(matching_domains) > 1: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") duplicate_domains.append(transition_domain_name) if len(matching_domain_informations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") missing_domain_informations.append(transition_domain_name) if len(matching_domain_invitations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") missing_domain_invites.append(transition_domain_name) total_missing_domains = len(missing_domains) @@ -174,7 +200,7 @@ class Command(BaseCommand): (These are transition domains which have duplicate entries in the Domain Table) {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} - {total_missing_domain_informations} Domains Information Entries missing: + {total_missing_domain_informations} Domain Information Entries missing: (These are transition domains which have no entries in the Domain Information Table) {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} @@ -185,79 +211,92 @@ class Command(BaseCommand): """ ) - def run_load_transition_domain_script(self, - file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - debug_max_entries_to_parse): - load_transition_domain_command_string = "./manage.py load_transition_domain " - load_transition_domain_command_string += file_location+domain_contacts_filename + " " - load_transition_domain_command_string += file_location+contacts_filename + " " - load_transition_domain_command_string += file_location+domain_statuses_filename + " " + def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: + """Prompts the user to inspect the given terminal command string + and asks if they wish to execute it. If the user responds (y), + execute the command""" - if sep is not None and sep != "|": - load_transition_domain_command_string += f"--sep {sep} " - - if reset_table: - load_transition_domain_command_string += "--resetTable " - - if debug_on: - load_transition_domain_command_string += "--debug " - - if debug_max_entries_to_parse > 0: - load_transition_domain_command_string += f"--limitParse {debug_max_entries_to_parse} " - - proceed_load_transition_domain = TerminalHelper.query_yes_no( - f"""{TerminalColors.OKCYAN} - ===================================== - Running load_transition_domain script - ===================================== - - {load_transition_domain_command_string} - {TerminalColors.FAIL} - Proceed? - {TerminalColors.ENDC}""" - ) - - if not proceed_load_transition_domain: - return - logger.info(f"""{TerminalColors.OKCYAN} - ==== EXECUTING... ==== - {TerminalColors.ENDC}""") - os.system(f"{load_transition_domain_command_string}") - - def run_transfer_script(self, debug_on): - command_string = "./manage.py transfer_transition_domains_to_domains " - - if debug_on: - command_string += "--debug " - - - proceed_load_transition_domain = TerminalHelper.query_yes_no( + # Allow the user to inspect the command string + # and ask if they wish to proceed + proceed_execution = TerminalHelper.query_yes_no( f"""{TerminalColors.OKCYAN} ===================================================== - Running transfer_transition_domains_to_domains script + {prompt_title} ===================================================== - + *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** + {command_string} {TerminalColors.FAIL} - Proceed? + Proceed? (Y = proceed, N = skip) {TerminalColors.ENDC}""" ) - if not proceed_load_transition_domain: - return + # If the user decided to proceed executing the command, + # run the command for loading transition domains. + # Otherwise, exit this subroutine. + if not proceed_execution: + sys.exit() + logger.info(f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== {TerminalColors.ENDC}""") os.system(f"{command_string}") + return True + + def run_load_transition_domain_script(self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + debug_max_entries_to_parse: int): + """Runs the load_transition_domain script""" + # Create the command string + command_string = "./manage.py load_transition_domain " + command_string += file_location+domain_contacts_filename + " " + command_string += file_location+contacts_filename + " " + command_string += file_location+domain_statuses_filename + " " + if sep is not None and sep != "|": + command_string += f"--sep {sep} " + if reset_table: + command_string += "--resetTable " + if debug_on: + command_string += "--debug " + if debug_max_entries_to_parse > 0: + command_string += f"--limitParse {debug_max_entries_to_parse} " + + # Execute the command string + self.prompt_for_execution(command_string, "Running load_transition_domain script") + + + def run_transfer_script(self, debug_on:bool): + """Runs the transfer_transition_domains_to_domains script""" + # Create the command string + command_string = "./manage.py transfer_transition_domains_to_domains " + if debug_on: + command_string += "--debug " + # Execute the command string + self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + + + def run_send_invites_script(self): + """Runs the send_domain_invitations script""" + # Create the command string... + command_string = "./manage.py send_domain_invitations -s" + # Execute the command string + self.prompt_for_execution(command_string, "Running send_domain_invitations script") + + def run_migration_scripts(self, options): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" + + # Grab filepath information from the arguments file_location = options.get("loaderDirectory")+"/" filenames = options.get("loaderFilenames").split() if len(filenames) < 3: @@ -277,6 +316,9 @@ class Command(BaseCommand): contacts_filename = filenames[1] domain_statuses_filename = filenames[2] + # Allow the user to inspect the filepath + # data given in the arguments, and prompt + # the user to verify this info before proceeding files_are_correct = TerminalHelper.query_yes_no( f""" {TerminalColors.OKCYAN} @@ -288,14 +330,17 @@ class Command(BaseCommand): ....for the following files: - domain contacts: {domain_contacts_filename} - contacts: {contacts_filename} - - domain statuses: {domain_statuses_filename}y + - domain statuses: {domain_statuses_filename} {TerminalColors.FAIL} Does this look correct?{TerminalColors.ENDC}""" ) + # If the user rejected the filepath information + # as incorrect, prompt the user to provide + # correct file inputs in their original command + # prompt and exit this subroutine if not files_are_correct: - # prompt the user to provide correct file inputs logger.info(f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: @@ -306,20 +351,16 @@ class Command(BaseCommand): """) return - # Get --sep argument + # Proceed executing the migration scripts... + # ...First, Get all the arguments sep = options.get("sep") - - # Get --resetTable argument reset_table = options.get("resetTable") - - # Get --debug argument debug_on = options.get("debug") - - # Get --limitParse argument debug_max_entries_to_parse = int( options.get("limitParse") - ) # set to 0 to parse all entries + ) + #...Second, Run the migration scripts in order self.run_load_transition_domain_script(file_location, domain_contacts_filename, contacts_filename, @@ -328,29 +369,49 @@ class Command(BaseCommand): reset_table, debug_on, debug_max_entries_to_parse) - self.run_transfer_script(debug_on) + def simulate_user_logins(self, debug_on): - logger.info(f"""{TerminalColors.OKCYAN} - ================== - SIMULATING LOGINS - ================== - {TerminalColors.ENDC} - """) - # for invite in DomainInvitation.objects.all(): - # #DEBUG: - # TerminalHelper.print_conditional(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") - # # get a user with this email address - # user_exists = User.objects.filter(email=invite.email).exists() - # user, _ = User.objects.get_or_create(email=invite.email) - # #DEBUG: - # TerminalHelper.print_conditional(user_exists,f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") - # TerminalHelper.print_conditional(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") + """Simulates logins for users (this will add + Domain Information objects to our tables)""" + + logger.info(f"" + f"{TerminalColors.OKCYAN}" + f"================== SIMULATING LOGINS ==================" + f"{TerminalColors.ENDC}") + for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff + #DEBUG: + TerminalHelper.print_conditional(debug_on, + f"{TerminalColors.OKCYAN}" + f"Processing invite: {invite}" + f"{TerminalColors.ENDC}") + # get a user with this email address + user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + #DEBUG: + TerminalHelper.print_conditional(user_created, + f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") + TerminalHelper.print_conditional(debug_on, + f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") + user.first_login() + if user_created: + logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + user.delete() + + # get a user with this email address + # user_exists = User.objects.filter(email=invite.email).exists() + # if user_exists: + # user = User.objects.get(email=invite.email) + # TerminalHelper.print_conditional(debug_on, + # f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") # user.first_login() - # if not user_exists: - # logger.warn(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") - # user.delete() + # else: + # logger.info(f"""{TerminalColors.YELLOW}No user found -- creating temp user object...{TerminalColors.ENDC}""") + # user = User(email=invite.email) + # user.save() + # user.first_login() + # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + # user.delete() # for invite in DomainInvitation.objects.all(): # #DEBUG: @@ -371,58 +432,108 @@ class Command(BaseCommand): **options, ): """ - Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. - - Produces the following report (printed to the terminal): - #1 - Print any domains that exist in the transition_domain table - but not in their corresponding domain, domain information or - domain invitation tables. - #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are - duplicates and in which tables - - (ONLY RUNS with full script option) - - Emails should be sent to the appropriate users - note that all moved domains should now be accessible - on django admin for an analyst - - OPTIONS: - -- (run all other scripts: - 1 - imports for trans domains - 2 - transfer to domain & domain invitation - 3 - send domain invite) - ** Triggers table reset ** + Does the following; + 1 - run loader scripts + 2 - simulate logins + 3 - send domain invitations (Emails should be sent to the appropriate users + note that all moved domains should now be accessible + on django admin for an analyst) + 4 - analyze the data for transition domains + and generate a report """ - # Get --debug argument + # SETUP + # Grab all arguments relevant to + # orchestrating which parts of this script + # should execute. Print some indicators to + # the terminal so the user knows what is + # enabled. debug_on = options.get("debug") - # Get --runLoaders argument run_loaders_on = options.get("runLoaders") - # Get --triggerLogins argument simulate_user_login_enabled = options.get("triggerLogins") - + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_on, + f"""{TerminalColors.OKCYAN} + ----------RUNNING LOADERS ON---------- + All migration scripts will be run before + analyzing the data. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_on, + f"""{TerminalColors.OKCYAN} + ----------TRIGGER LOGINS ON---------- + Will be simulating user logins + {TerminalColors.ENDC} + """ + ) + # If a user decides to run all migration + # scripts, they may or may not wish to + # proceed with analysis of the data depending + # on the results of the migration. + # Provide a breakpoint for them to decide + # whether to continue or not. + # The same will happen if simulating user + # logins (to allow users to run only that + # portion of the script if desired) prompt_continuation_of_analysis = False - # Run migration scripts if specified by user... + # STEP 1 -- RUN LOADERS + # Run migration scripts if specified by user if run_loaders_on: self.run_migration_scripts(options) prompt_continuation_of_analysis = True - # Simulate user login for each user in domain invitation if sepcified by user + # STEP 2 -- SIMULATE LOGINS + # Simulate user login for each user in domain + # invitation if specified by user OR if running + # migration scripts. + # (NOTE: Although users can choose to run login + # simulations separately (for testing purposes), + # if we are running all migration scripts, we should + # automatically execute this as the final step + # to ensure Domain Information objects get added + # to the database.) + if run_loaders_on: + simulate_user_login_enabled = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with simulating user logins? + {TerminalColors.ENDC}""" + ) if simulate_user_login_enabled: self.simulate_user_logins(debug_on) prompt_continuation_of_analysis = True - analyze_tables = True + + # STEP 3 -- SEND INVITES + proceed_with_sending_invites = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with sending user invites? + {TerminalColors.ENDC}""" + ) + if proceed_with_sending_invites: + self.run_send_invites_script() + prompt_continuation_of_analysis = True + + # STEP 4 -- ANALYZE TABLES & GENERATE REPORT + # Analyze tables for corrupt data... + + # only prompt if we ran steps 1 and/or 2 if prompt_continuation_of_analysis: analyze_tables = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with table analysis? {TerminalColors.ENDC}""" ) - - # Analyze tables for corrupt data... - if analyze_tables: - self.compare_tables(debug_on) \ No newline at end of file + if not analyze_tables: + return + self.compare_tables(debug_on) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index b98e8e2a9..9443f085f 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -10,25 +10,14 @@ from registrar.models import TransitionDomain from registrar.models import Domain from registrar.models import DomainInvitation +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper +) + logger = logging.getLogger(__name__) -class termColors: - """Colors for terminal outputs - (makes reading the logs WAY easier)""" - - HEADER = "\033[95m" - OKBLUE = "\033[94m" - OKCYAN = "\033[96m" - OKGREEN = "\033[92m" - YELLOW = "\033[93m" - FAIL = "\033[91m" - ENDC = "\033[0m" - BOLD = "\033[1m" - UNDERLINE = "\033[4m" - BackgroundLightYellow = "\033[103m" - - class Command(BaseCommand): help = """Load data from transition domain tables into main domain tables. Also create domain invitation @@ -49,33 +38,25 @@ class Command(BaseCommand): ): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. - {termColors.ENDC} + {TerminalColors.ENDC} """, ) - self.print_debug( + TerminalHelper.print_conditional( debug_max_entries_to_parse > 0, - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} ----------LIMITER ON---------- Parsing of entries will be limited to {debug_max_entries_to_parse} lines per file.") Detailed print statements activated. - {termColors.ENDC} + {TerminalColors.ENDC} """, ) - def print_debug(self, print_condition: bool, print_statement: str): - """This function reduces complexity of debug statements - in other functions. - It uses the logger to write the given print_statement to the - terminal if print_condition is TRUE""" - # DEBUG: - if print_condition: - logger.info(print_statement) def update_domain_status( self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool @@ -96,13 +77,13 @@ class Command(BaseCommand): target_domain.save() # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} >> Updated {target_domain.name} state from '{existing_status}' to '{target_domain.state}' (no domain invitation entry added) - {termColors.ENDC}""", + {TerminalColors.ENDC}""", ) return True return False @@ -123,22 +104,22 @@ class Command(BaseCommand): total_domain_invitation_entries = len(domain_invitations_to_create) logger.info( - f"""{termColors.OKGREEN} + f"""{TerminalColors.OKGREEN} ============= FINISHED =============== Created {total_new_entries} transition domain entries, Updated {total_updated_domain_entries} transition domain entries Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) - {termColors.ENDC} + {TerminalColors.ENDC} """ ) if len(skipped_domain_entries) > 0: logger.info( - f"""{termColors.FAIL} + f"""{TerminalColors.FAIL} ============= SKIPPED DOMAINS (ERRORS) =============== {skipped_domain_entries} - {termColors.ENDC} + {TerminalColors.ENDC} """ ) @@ -151,25 +132,25 @@ class Command(BaseCommand): skipped_domain_invitations.remove(domain_invite.domain) if len(skipped_domain_invitations) > 0: logger.info( - f"""{termColors.FAIL} + f"""{TerminalColors.FAIL} ============= SKIPPED DOMAIN INVITATIONS (ERRORS) =============== {skipped_domain_invitations} - {termColors.ENDC} + {TerminalColors.ENDC} """ ) # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.YELLOW} - + f"""{TerminalColors.YELLOW} + ======= DEBUG OUTPUT ======= Created Domains: {domains_to_create} Updated Domains: {updated_domain_entries} - {termColors.ENDC} + {TerminalColors.ENDC} """, ) @@ -184,7 +165,7 @@ class Command(BaseCommand): if associated_domain is None: logger.warning( f""" - {termColors.FAIL} + {TerminalColors.FAIL} !!! ERROR: Domain cannot be null for a Domain Invitation object! @@ -241,11 +222,11 @@ class Command(BaseCommand): total_rows_parsed = 0 logger.info( - f"""{termColors.OKGREEN} + f"""{TerminalColors.OKGREEN} ========================== Beginning Data Transfer ========================== - {termColors.ENDC}""" + {TerminalColors.ENDC}""" ) for transition_domain in TransitionDomain.objects.all(): @@ -254,11 +235,11 @@ class Command(BaseCommand): transition_domain_email = transition_domain.username # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.OKCYAN} + f"""{TerminalColors.OKCYAN} Processing Transition Domain: {transition_domain_name}, {transition_domain_status}, {transition_domain_email} - {termColors.ENDC}""", # noqa + {TerminalColors.ENDC}""", # noqa ) new_domain_invitation = None @@ -269,11 +250,11 @@ class Command(BaseCommand): # get the existing domain domain_to_update = Domain.objects.get(name=transition_domain_name) # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} > Found existing entry in Domain table for: {transition_domain_name}, {domain_to_update.state} - {termColors.ENDC}""", # noqa + {TerminalColors.ENDC}""", # noqa ) # for existing entry, update the status to @@ -300,7 +281,7 @@ class Command(BaseCommand): # immediate attention. logger.warning( f""" - {termColors.FAIL} + {TerminalColors.FAIL} !!! ERROR: duplicate entries already exist in the Domain table for the following domain: {transition_domain_name} @@ -316,7 +297,7 @@ class Command(BaseCommand): except TransitionNotAllowed as err: skipped_domain_entries.append(transition_domain_name) logger.warning( - f"""{termColors.FAIL} + f"""{TerminalColors.FAIL} Unable to change state for {transition_domain_name} RECOMMENDATION: @@ -343,15 +324,15 @@ class Command(BaseCommand): None, ) if existing_domain_in_to_create is not None: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} Duplicate Detected: {transition_domain_name}. Cannot add duplicate entry for another username. Violates Unique Key constraint. Checking for unique user e-mail for Domain Invitations... - {termColors.ENDC}""", + {TerminalColors.ENDC}""", ) new_domain_invitation = self.try_add_domain_invitation( transition_domain_email, existing_domain_in_to_create @@ -363,9 +344,9 @@ class Command(BaseCommand): ) domains_to_create.append(new_domain) # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"{termColors.OKCYAN} Adding domain: {new_domain} {termColors.ENDC}", # noqa + f"{TerminalColors.OKCYAN} Adding domain: {new_domain} {TerminalColors.ENDC}", # noqa ) new_domain_invitation = self.try_add_domain_invitation( transition_domain_email, new_domain @@ -373,14 +354,14 @@ class Command(BaseCommand): if new_domain_invitation is None: logger.info( - f"{termColors.YELLOW} ! No new e-mail detected !" # noqa - f"(SKIPPED ADDING DOMAIN INVITATION){termColors.ENDC}" + f"{TerminalColors.YELLOW} ! No new e-mail detected !" # noqa + f"(SKIPPED ADDING DOMAIN INVITATION){TerminalColors.ENDC}" ) else: # DEBUG: - self.print_debug( + TerminalHelper.print_conditional( debug_on, - f"{termColors.OKCYAN} Adding domain invitation: {new_domain_invitation} {termColors.ENDC}", # noqa + f"{TerminalColors.OKCYAN} Adding domain invitation: {new_domain_invitation} {TerminalColors.ENDC}", # noqa ) domain_invitations_to_create.append(new_domain_invitation) @@ -390,9 +371,9 @@ class Command(BaseCommand): and total_rows_parsed >= debug_max_entries_to_parse ): logger.info( - f"""{termColors.YELLOW} + f"""{TerminalColors.YELLOW} ----PARSE LIMIT REACHED. HALTING PARSER.---- - {termColors.ENDC} + {TerminalColors.ENDC} """ ) break diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index abfcdcae0..3f647f0b8 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -11,6 +11,7 @@ class TerminalColors: OKCYAN = "\033[96m" OKGREEN = "\033[92m" YELLOW = "\033[93m" + MAGENTA = "\033[35m" FAIL = "\033[91m" ENDC = "\033[0m" BOLD = "\033[1m" From d5e9e45a2fb7f72705523c28214e4d4bad3de47c Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 18:37:58 -0600 Subject: [PATCH 11/49] added logic to bypass prompts if debug is enabled --- .../commands/test_domain_migration.py | 229 +++++++++--------- 1 file changed, 112 insertions(+), 117 deletions(-) diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py index d71f88c36..781de38e0 100644 --- a/src/registrar/management/commands/test_domain_migration.py +++ b/src/registrar/management/commands/test_domain_migration.py @@ -237,12 +237,17 @@ class Command(BaseCommand): if not proceed_execution: sys.exit() + self.execute_command(command_string) + + return True + + def execute_command(self, command_string:str): + """Executes the given command string""" + logger.info(f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== {TerminalColors.ENDC}""") os.system(f"{command_string}") - - return True def run_load_transition_domain_script(self, file_location: str, @@ -252,6 +257,7 @@ class Command(BaseCommand): sep: str, reset_table: bool, debug_on: bool, + prompts_enabled: bool, debug_max_entries_to_parse: int): """Runs the load_transition_domain script""" # Create the command string @@ -269,33 +275,51 @@ class Command(BaseCommand): command_string += f"--limitParse {debug_max_entries_to_parse} " # Execute the command string - self.prompt_for_execution(command_string, "Running load_transition_domain script") + if prompts_enabled: + self.prompt_for_execution(command_string, "Running load_transition_domain script") + return + self.execute_command(command_string) - def run_transfer_script(self, debug_on:bool): + def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" # Create the command string command_string = "./manage.py transfer_transition_domains_to_domains " if debug_on: command_string += "--debug " # Execute the command string - self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + if prompts_enabled: + self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + return + self.execute_command(command_string) - def run_send_invites_script(self): + def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" # Create the command string... command_string = "./manage.py send_domain_invitations -s" # Execute the command string - self.prompt_for_execution(command_string, "Running send_domain_invitations script") + if prompts_enabled: + self.prompt_for_execution(command_string, "Running send_domain_invitations script") + return + self.execute_command(command_string) def run_migration_scripts(self, + prompts_enabled: bool, options): """Runs the following migration scripts (in order): 1 - imports for trans domains 2 - transfer to domain & domain invitation""" + # Get arguments + sep = options.get("sep") + reset_table = options.get("resetTable") + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) + # Grab filepath information from the arguments file_location = options.get("loaderDirectory")+"/" filenames = options.get("loaderFilenames").split() @@ -311,56 +335,48 @@ class Command(BaseCommand): ============= TERMINATING ============= {TerminalColors.ENDC} """) - return + sys.exit() domain_contacts_filename = filenames[0] contacts_filename = filenames[1] domain_statuses_filename = filenames[2] - # Allow the user to inspect the filepath - # data given in the arguments, and prompt - # the user to verify this info before proceeding - files_are_correct = TerminalHelper.query_yes_no( - f""" - {TerminalColors.OKCYAN} - *** IMPORTANT: VERIFY THE FOLLOWING *** + if prompts_enabled: + # Allow the user to inspect the filepath + # data given in the arguments, and prompt + # the user to verify this info before proceeding + files_are_correct = TerminalHelper.query_yes_no( + f""" + {TerminalColors.OKCYAN} + *** IMPORTANT: VERIFY THE FOLLOWING *** - The migration scripts are looking in directory.... - {file_location} + The migration scripts are looking in directory.... + {file_location} - ....for the following files: - - domain contacts: {domain_contacts_filename} - - contacts: {contacts_filename} - - domain statuses: {domain_statuses_filename} + ....for the following files: + - domain contacts: {domain_contacts_filename} + - contacts: {contacts_filename} + - domain statuses: {domain_statuses_filename} - {TerminalColors.FAIL} - Does this look correct?{TerminalColors.ENDC}""" - ) + {TerminalColors.FAIL} + Does this look correct?{TerminalColors.ENDC}""" + ) - # If the user rejected the filepath information - # as incorrect, prompt the user to provide - # correct file inputs in their original command - # prompt and exit this subroutine - if not files_are_correct: - logger.info(f""" - {TerminalColors.YELLOW} - PLEASE Re-Run the script with the correct file location and filenames: - - EXAMPLE: - docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - - """) - return + # If the user rejected the filepath information + # as incorrect, prompt the user to provide + # correct file inputs in their original command + # prompt and exit this subroutine + if not files_are_correct: + logger.info(f""" + {TerminalColors.YELLOW} + PLEASE Re-Run the script with the correct file location and filenames: + + EXAMPLE: + docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt + + """) + return - # Proceed executing the migration scripts... - # ...First, Get all the arguments - sep = options.get("sep") - reset_table = options.get("resetTable") - debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) - - #...Second, Run the migration scripts in order + # Proceed executing the migration scripts self.run_load_transition_domain_script(file_location, domain_contacts_filename, contacts_filename, @@ -368,8 +384,9 @@ class Command(BaseCommand): sep, reset_table, debug_on, + prompts_enabled, debug_max_entries_to_parse) - self.run_transfer_script(debug_on) + self.run_transfer_script(debug_on, prompts_enabled) def simulate_user_logins(self, debug_on): @@ -380,52 +397,27 @@ class Command(BaseCommand): f"{TerminalColors.OKCYAN}" f"================== SIMULATING LOGINS ==================" f"{TerminalColors.ENDC}") - for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff - #DEBUG: - TerminalHelper.print_conditional(debug_on, - f"{TerminalColors.OKCYAN}" - f"Processing invite: {invite}" - f"{TerminalColors.ENDC}") - # get a user with this email address - user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) - #DEBUG: - TerminalHelper.print_conditional(user_created, - f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") - TerminalHelper.print_conditional(debug_on, - f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") - user.first_login() - if user_created: - logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") - user.delete() + - # get a user with this email address - # user_exists = User.objects.filter(email=invite.email).exists() - # if user_exists: - # user = User.objects.get(email=invite.email) - # TerminalHelper.print_conditional(debug_on, - # f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") - # user.first_login() - # else: - # logger.info(f"""{TerminalColors.YELLOW}No user found -- creating temp user object...{TerminalColors.ENDC}""") - # user = User(email=invite.email) - # user.save() - # user.first_login() - # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") - # user.delete() - - # for invite in DomainInvitation.objects.all(): + + # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff # #DEBUG: - # TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Processing invite: {invite}{TerminalColors.ENDC}""") - # # get a user with this email address - # User = get_user_model() - # try: - # user = User.objects.get(email=invite.email) - # #DEBUG: - # TerminalHelper.print_debug(debug_on,f"""{TerminalColors.OKCYAN}Logging in user: {user}{TerminalColors.ENDC}""") - # Client.force_login(user) - # except User.DoesNotExist: - # #TODO: how should we handle this? - # logger.warn(f"""{TerminalColors.FAIL}No user found {invite.email}{TerminalColors.ENDC}""") + # TerminalHelper.print_conditional(debug_on, + # f"{TerminalColors.OKCYAN}" + # f"Processing invite: {invite}" + # f"{TerminalColors.ENDC}") + # # get a user with this email address + # user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + # #DEBUG: + # TerminalHelper.print_conditional(user_created, + # f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") + # TerminalHelper.print_conditional(debug_on, + # f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") + # user.first_login() + # if user_created: + # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + # user.delete() + def handle( self, @@ -449,7 +441,8 @@ class Command(BaseCommand): # the terminal so the user knows what is # enabled. debug_on = options.get("debug") - run_loaders_on = options.get("runLoaders") + prompts_enabled = debug_on #TODO: add as argument? + run_loaders_enabled = options.get("runLoaders") simulate_user_login_enabled = options.get("triggerLogins") TerminalHelper.print_conditional( debug_on, @@ -460,7 +453,7 @@ class Command(BaseCommand): """ ) TerminalHelper.print_conditional( - run_loaders_on, + run_loaders_enabled, f"""{TerminalColors.OKCYAN} ----------RUNNING LOADERS ON---------- All migration scripts will be run before @@ -469,7 +462,7 @@ class Command(BaseCommand): """ ) TerminalHelper.print_conditional( - run_loaders_on, + run_loaders_enabled, f"""{TerminalColors.OKCYAN} ----------TRIGGER LOGINS ON---------- Will be simulating user logins @@ -489,8 +482,8 @@ class Command(BaseCommand): # STEP 1 -- RUN LOADERS # Run migration scripts if specified by user - if run_loaders_on: - self.run_migration_scripts(options) + if run_loaders_enabled: + self.run_migration_scripts(options, prompts_enabled) prompt_continuation_of_analysis = True # STEP 2 -- SIMULATE LOGINS @@ -503,32 +496,34 @@ class Command(BaseCommand): # automatically execute this as the final step # to ensure Domain Information objects get added # to the database.) - if run_loaders_on: - simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with simulating user logins? - {TerminalColors.ENDC}""" - ) - if simulate_user_login_enabled: + if run_loaders_enabled: + if prompts_enabled: + simulate_user_login_enabled = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with simulating user logins? + {TerminalColors.ENDC}""" + ) + if not simulate_user_login_enabled: + return self.simulate_user_logins(debug_on) prompt_continuation_of_analysis = True - # STEP 3 -- SEND INVITES - proceed_with_sending_invites = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with sending user invites? - {TerminalColors.ENDC}""" - ) - if proceed_with_sending_invites: - self.run_send_invites_script() - prompt_continuation_of_analysis = True + if prompts_enabled: + proceed_with_sending_invites = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with sending user invites? + {TerminalColors.ENDC}""" + ) + if not proceed_with_sending_invites: + return + self.run_send_invites_script(debug_on) + prompt_continuation_of_analysis = True # STEP 4 -- ANALYZE TABLES & GENERATE REPORT # Analyze tables for corrupt data... - - # only prompt if we ran steps 1 and/or 2 - if prompt_continuation_of_analysis: + if prompt_continuation_of_analysis & prompts_enabled: + # ^ (only prompt if we ran steps 1 and/or 2) analyze_tables = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with table analysis? From 02ec4d7bae7127fe39ed3db883ab4f194ac229a0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 24 Oct 2023 19:45:16 -0600 Subject: [PATCH 12/49] Renamed main script and added test script (to separate concerns of testing vs combining transition domain migration scripts). Added test data files. --- .../commands/full_domain_migrations.py | 532 ++++++++++++++++++ src/registrar/tests/data/test_contacts.txt | 7 + .../tests/data/test_domain_contacts.txt | 20 + .../tests/data/test_domain_statuses.txt | 6 + ...ransition_domain_migrations_with_logins.py | 42 ++ 5 files changed, 607 insertions(+) create mode 100644 src/registrar/management/commands/full_domain_migrations.py create mode 100644 src/registrar/tests/data/test_contacts.txt create mode 100644 src/registrar/tests/data/test_domain_contacts.txt create mode 100644 src/registrar/tests/data/test_domain_statuses.txt create mode 100644 src/registrar/tests/test_transition_domain_migrations_with_logins.py diff --git a/src/registrar/management/commands/full_domain_migrations.py b/src/registrar/management/commands/full_domain_migrations.py new file mode 100644 index 000000000..7e47e8075 --- /dev/null +++ b/src/registrar/management/commands/full_domain_migrations.py @@ -0,0 +1,532 @@ +"""Data migration: + 1 - generates a report of data integrity across all + transition domain related tables + 2 - allows users to run all migration scripts for + transition domain data +""" + +import logging +import argparse +import sys +import os + +from django.test import Client + +from django_fsm import TransitionNotAllowed # type: ignore + +from django.core.management import BaseCommand + +from registrar.models import ( + Domain, + DomainInformation, + DomainInvitation, + TransitionDomain, + User, +) + +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper +) + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = """ """ + + def add_arguments(self, parser): + """ + OPTIONAL ARGUMENTS: + --runLoaders + A boolean (default to true), which triggers running + all scripts (in sequence) for transition domain migrations + + --triggerLogins + A boolean (default to true), which triggers running + simulations of user logins for each user in domain invitation + + --loaderDirectory + The location of the files used for load_transition_domain migration script + EXAMPLE USAGE: + > --loaderDirectory /app/tmp + + --loaderFilenames + The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + EXAMPLE USAGE: + > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information + + --sep + Delimiter for the loaders to correctly parse the given text files. + (usually this can remain at default value of |) + + --debug + A boolean (default to true), which activates additional print statements + + --limitParse + Used by the loaders (load_transition_domain) to set the limit for the + number of data entries to insert. Set to 0 (or just don't use this + argument) to parse every entry. This was provided primarily for testing + purposes + + --resetTable + Used by the loaders to trigger a prompt for deleting all table entries. + Useful for testing purposes, but USE WITH CAUTION + """ + + parser.add_argument("--runLoaders", + help="Runs all scripts (in sequence) for transition domain migrations", + action=argparse.BooleanOptionalAction) + + parser.add_argument("--triggerLogins", + help="Simulates a user login for each user in domain invitation", + action=argparse.BooleanOptionalAction) + + # The following file arguments have default values for running in the sandbox + parser.add_argument( + "--loaderDirectory", + default="migrationData", + help="The location of the files used for load_transition_domain migration script" + ) + parser.add_argument( + "--loaderFilenames", + default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: + domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + + where... + - domain_contacts_filename is the Data file with domain contact information + - contacts_filename is the Data file with contact information + - domain_statuses_filename is the Data file with domain status information""" + ) + + parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") + + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", default=0, help="Sets max number of entries to load" + ) + + parser.add_argument( + "--resetTable", + help="Deletes all data in the TransitionDomain table", + action=argparse.BooleanOptionalAction, + ) + + + + def compare_tables(self, debug_on: bool): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + + Produces the following report (printed to the terminal): + #1 - Print any domains that exist in the transition_domain table + but not in their corresponding domain, domain information or + domain invitation tables. + #2 - Print which table this domain is missing from + #3- Check for duplicate entries in domain or + domain_information tables and print which are + duplicates and in which tables + """ + + logger.info( + f"""{TerminalColors.OKCYAN} + ============= BEGINNING ANALYSIS =============== + {TerminalColors.ENDC} + """ + ) + + #TODO: would filteredRelation be faster? + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] + for transition_domain in TransitionDomain.objects.all():# DEBUG: + transition_domain_name = transition_domain.domain_name + transition_domain_email = transition_domain.username + + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN}Checking: {transition_domain_name} {TerminalColors.ENDC}", # noqa + ) + + # Check Domain table + matching_domains = Domain.objects.filter(name=transition_domain_name) + # Check Domain Information table + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + # Check Domain Invitation table + matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), + domain__name=transition_domain_name) + + if len(matching_domains) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") + missing_domains.append(transition_domain_name) + elif len(matching_domains) > 1: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") + duplicate_domains.append(transition_domain_name) + if len(matching_domain_informations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") + missing_domain_informations.append(transition_domain_name) + if len(matching_domain_invitations) == 0: + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") + missing_domain_invites.append(transition_domain_name) + + total_missing_domains = len(missing_domains) + total_duplicate_domains = len(duplicate_domains) + total_missing_domain_informations = len(missing_domain_informations) + total_missing_domain_invitations = len(missing_domain_invites) + + missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) + duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) + missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) + missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) + + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED ANALYSIS =============== + + {total_missing_domains} Missing Domains: + (These are transition domains that are missing from the Domain Table) + {TerminalColors.YELLOW}{missing_domains_as_string}{TerminalColors.OKGREEN} + + {total_duplicate_domains} Duplicate Domains: + (These are transition domains which have duplicate entries in the Domain Table) + {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_informations} Domain Information Entries missing: + (These are transition domains which have no entries in the Domain Information Table) + {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} + + {total_missing_domain_invitations} Domain Invitations missing: + (These are transition domains which have no entires in the Domain Invitation Table) + {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} + {TerminalColors.ENDC} + """ + ) + + def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: + """Prompts the user to inspect the given terminal command string + and asks if they wish to execute it. If the user responds (y), + execute the command""" + + # Allow the user to inspect the command string + # and ask if they wish to proceed + proceed_execution = TerminalHelper.query_yes_no( + f"""{TerminalColors.OKCYAN} + ===================================================== + {prompt_title} + ===================================================== + *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** + + {command_string} + {TerminalColors.FAIL} + Proceed? (Y = proceed, N = skip) + {TerminalColors.ENDC}""" + ) + + # If the user decided to proceed executing the command, + # run the command for loading transition domains. + # Otherwise, exit this subroutine. + if not proceed_execution: + sys.exit() + + self.execute_command(command_string) + + return True + + def execute_command(self, command_string:str): + """Executes the given command string""" + + logger.info(f"""{TerminalColors.OKCYAN} + ==== EXECUTING... ==== + {TerminalColors.ENDC}""") + os.system(f"{command_string}") + + def run_load_transition_domain_script(self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int): + """Runs the load_transition_domain script""" + # Create the command string + command_string = "./manage.py load_transition_domain " + command_string += file_location+domain_contacts_filename + " " + command_string += file_location+contacts_filename + " " + command_string += file_location+domain_statuses_filename + " " + if sep is not None and sep != "|": + command_string += f"--sep {sep} " + if reset_table: + command_string += "--resetTable " + if debug_on: + command_string += "--debug " + if debug_max_entries_to_parse > 0: + command_string += f"--limitParse {debug_max_entries_to_parse} " + + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running load_transition_domain script") + return + self.execute_command(command_string) + + + def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): + """Runs the transfer_transition_domains_to_domains script""" + # Create the command string + command_string = "./manage.py transfer_transition_domains_to_domains " + if debug_on: + command_string += "--debug " + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + return + self.execute_command(command_string) + + + def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): + """Runs the send_domain_invitations script""" + # Create the command string... + command_string = "./manage.py send_domain_invitations -s" + # Execute the command string + if prompts_enabled: + self.prompt_for_execution(command_string, "Running send_domain_invitations script") + return + self.execute_command(command_string) + + + def run_migration_scripts(self, + prompts_enabled: bool, + options): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" + + # Get arguments + sep = options.get("sep") + reset_table = options.get("resetTable") + debug_on = options.get("debug") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) + + # Grab filepath information from the arguments + file_location = options.get("loaderDirectory")+"/" + filenames = options.get("loaderFilenames").split() + if len(filenames) < 3: + filenames_as_string = "{}".format(", ".join(map(str, filenames))) + logger.info(f""" + {TerminalColors.FAIL} + --loaderFilenames expected 3 filenames to follow it, + but only {len(filenames)} were given: + {filenames_as_string} + + PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN + ============= TERMINATING ============= + {TerminalColors.ENDC} + """) + sys.exit() + domain_contacts_filename = filenames[0] + contacts_filename = filenames[1] + domain_statuses_filename = filenames[2] + + if prompts_enabled: + # Allow the user to inspect the filepath + # data given in the arguments, and prompt + # the user to verify this info before proceeding + files_are_correct = TerminalHelper.query_yes_no( + f""" + {TerminalColors.OKCYAN} + *** IMPORTANT: VERIFY THE FOLLOWING *** + + The migration scripts are looking in directory.... + {file_location} + + ....for the following files: + - domain contacts: {domain_contacts_filename} + - contacts: {contacts_filename} + - domain statuses: {domain_statuses_filename} + + {TerminalColors.FAIL} + Does this look correct?{TerminalColors.ENDC}""" + ) + + # If the user rejected the filepath information + # as incorrect, prompt the user to provide + # correct file inputs in their original command + # prompt and exit this subroutine + if not files_are_correct: + logger.info(f""" + {TerminalColors.YELLOW} + PLEASE Re-Run the script with the correct file location and filenames: + + EXAMPLE: + docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt + + """) + return + + # Proceed executing the migration scripts + self.run_load_transition_domain_script(file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse) + self.run_transfer_script(debug_on, prompts_enabled) + + + def simulate_user_logins(self, debug_on): + """Simulates logins for users (this will add + Domain Information objects to our tables)""" + + logger.info(f"" + f"{TerminalColors.OKCYAN}" + f"================== SIMULATING LOGINS ==================" + f"{TerminalColors.ENDC}") + + # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff + # #DEBUG: + # TerminalHelper.print_conditional(debug_on, + # f"{TerminalColors.OKCYAN}" + # f"Processing invite: {invite}" + # f"{TerminalColors.ENDC}") + # # get a user with this email address + # user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + # #DEBUG: + # TerminalHelper.print_conditional(user_created, + # f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") + # TerminalHelper.print_conditional(debug_on, + # f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") + # user.first_login() + # if user_created: + # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") + # user.delete() + + + def handle( + self, + **options, + ): + """ + Does the following; + 1 - run loader scripts + 2 - simulate logins + 3 - send domain invitations (Emails should be sent to the appropriate users + note that all moved domains should now be accessible + on django admin for an analyst) + 4 - analyze the data for transition domains + and generate a report + """ + + # SETUP + # Grab all arguments relevant to + # orchestrating which parts of this script + # should execute. Print some indicators to + # the terminal so the user knows what is + # enabled. + debug_on = options.get("debug") + prompts_enabled = debug_on #TODO: add as argument? + run_loaders_enabled = options.get("runLoaders") + simulate_user_login_enabled = options.get("triggerLogins") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} + ----------RUNNING LOADERS ON---------- + All migration scripts will be run before + analyzing the data. + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} + ----------TRIGGER LOGINS ON---------- + Will be simulating user logins + {TerminalColors.ENDC} + """ + ) + # If a user decides to run all migration + # scripts, they may or may not wish to + # proceed with analysis of the data depending + # on the results of the migration. + # Provide a breakpoint for them to decide + # whether to continue or not. + # The same will happen if simulating user + # logins (to allow users to run only that + # portion of the script if desired) + prompt_continuation_of_analysis = False + + # STEP 1 -- RUN LOADERS + # Run migration scripts if specified by user + if run_loaders_enabled: + self.run_migration_scripts(options, prompts_enabled) + prompt_continuation_of_analysis = True + + # STEP 2 -- SIMULATE LOGINS + # Simulate user login for each user in domain + # invitation if specified by user OR if running + # migration scripts. + # (NOTE: Although users can choose to run login + # simulations separately (for testing purposes), + # if we are running all migration scripts, we should + # automatically execute this as the final step + # to ensure Domain Information objects get added + # to the database.) + if run_loaders_enabled: + if prompts_enabled: + simulate_user_login_enabled = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with simulating user logins? + {TerminalColors.ENDC}""" + ) + if not simulate_user_login_enabled: + return + self.simulate_user_logins(debug_on) + prompt_continuation_of_analysis = True + + # STEP 3 -- SEND INVITES + if prompts_enabled: + proceed_with_sending_invites = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with sending user invites? + {TerminalColors.ENDC}""" + ) + if not proceed_with_sending_invites: + return + self.run_send_invites_script(debug_on) + prompt_continuation_of_analysis = True + + # STEP 4 -- ANALYZE TABLES & GENERATE REPORT + # Analyze tables for corrupt data... + if prompt_continuation_of_analysis & prompts_enabled: + # ^ (only prompt if we ran steps 1 and/or 2) + analyze_tables = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with table analysis? + {TerminalColors.ENDC}""" + ) + if not analyze_tables: + return + self.compare_tables(debug_on) diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt new file mode 100644 index 000000000..dec8f6816 --- /dev/null +++ b/src/registrar/tests/data/test_contacts.txt @@ -0,0 +1,7 @@ +TESTUSER|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||testuser@gmail.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| +RJD1|52545_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||agustina.wyman7@test.com|GSA|VERISIGN|ctldbatch|2021-06-29T18:53:09Z|VERISIGN|ctldbatch|2021-06-29T18:58:08Z| +JAKING|52555_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||susy.martin4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T15:38:10Z| +JBOONE|52556_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||stephania.winters4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T18:28:09Z| +MKELLEY|52557_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||alexandra.bobbitt5@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-08-02T22:13:09Z| +CWILSON|52562_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||jospeh.mcdowell3@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:33:09Z| +LMCCADE|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||reginald.ratcliff4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt new file mode 100644 index 000000000..c25181c28 --- /dev/null +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -0,0 +1,20 @@ +Anomaly.gov|ANOMALY|tech +TestDomain.gov|TESTUSER|admin +NEHRP.GOV|RJD1|admin +NEHRP.GOV|JAKING|tech +NEHRP.GOV|JBOONE|billing +NELSONCOUNTY-VA.GOV|MKELLEY|admin +NELSONCOUNTY-VA.GOV|CWILSON|billing +NELSONCOUNTY-VA.GOV|LMCCADE|tech +NELSONVILLENY.GOV|MBOWMAN|tech +NELSONVILLENY.GOV|PMINNERS|billing +NELSONVILLENY.GOV|CWINWARD|admin +NEMI.GOV|DJS1|tech +NEMI.GOV|EREAD|admin +NEMI.GOV|CHOPKINS|billing +NEPA.GOV|BPAUWELS|admin +NEPA.GOV|CRIBEIRO|billing +NEPA.GOV|DKAUFMAN1|tech +NERSC.GOV|TONEILL|admin +NERSC.GOV|JGERSTLE|billing +NERSC.GOV|RSTROMSNESS|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt new file mode 100644 index 000000000..60d526f2d --- /dev/null +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -0,0 +1,6 @@ +Anomaly.gov|muahaha| +TestDomain.gov|ok| +NEHRP.GOV|serverHold| +NELSONCOUNTY-VA.GOV|Hold| +NEMI.GOV|clientHold| +NERSC.GOV|ok| \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations_with_logins.py b/src/registrar/tests/test_transition_domain_migrations_with_logins.py new file mode 100644 index 000000000..2b3d7b6e7 --- /dev/null +++ b/src/registrar/tests/test_transition_domain_migrations_with_logins.py @@ -0,0 +1,42 @@ +from django.test import TestCase +from django.db.utils import IntegrityError +from unittest.mock import patch + +from registrar.models import ( + Contact, + DomainApplication, + DomainInformation, + User, + Website, + Domain, + DraftDomain, + DomainInvitation, + UserDomainRole, +) + +import boto3_mocking # type: ignore +from .common import MockSESClient, less_console_noise, completed_application +from django_fsm import TransitionNotAllowed + +boto3_mocking.clients.register_handler("sesv2", MockSESClient) + +@boto3_mocking.patching +class TestInvitations(TestCase): + + """Test the retrieval of invitations.""" + + def setUp(self): + self.domain, _ = Domain.objects.get_or_create(name="igorville.gov") + self.email = "mayor@igorville.gov" + self.invitation, _ = DomainInvitation.objects.get_or_create( + email=self.email, domain=self.domain + ) + self.user, _ = User.objects.get_or_create(email=self.email) + + # clean out the roles each time + UserDomainRole.objects.all().delete() + + def test_user_logins(self): + """A new user's first_login callback retrieves their invitations.""" + self.user.first_login() + self.assertTrue(UserDomainRole.objects.get(user=self.user, domain=self.domain)) From 53c74a766f6c3d797ff6b5f71f22ed09e7478a3e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:06:36 -0600 Subject: [PATCH 13/49] Linter things Removed old test_domain_migration.py file, and linted many items. --- .../commands/cat_files_into_getgov.py | 31 +- .../commands/full_domain_migrations.py | 294 +++++----- .../commands/load_transition_domain.py | 2 +- .../commands/test_domain_migration.py | 534 ------------------ .../transfer_transition_domains_to_domains.py | 3 +- .../commands/utility/terminal_helper.py | 5 +- ...ransition_domain_migrations_with_logins.py | 13 - 7 files changed, 190 insertions(+), 692 deletions(-) delete mode 100644 src/registrar/management/commands/test_domain_migration.py diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 9111263a6..5de44ba73 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -1,6 +1,5 @@ """Loads files from /tmp into our sandboxes""" import glob -import csv import logging import os @@ -21,17 +20,19 @@ class Command(BaseCommand): default="txt", help="What file extensions to look for, like txt or gz", ) - parser.add_argument("--directory", default="migrationdata", help="Desired directory") + parser.add_argument( + "--directory", default="migrationdata", help="Desired directory" + ) def handle(self, **options): - file_extension: str = options.get("file_extension").lstrip('.') + file_extension: str = options.get("file_extension").lstrip(".") directory = options.get("directory") # file_extension is always coerced as str, Truthy is OK to use here. if not file_extension or not isinstance(file_extension, str): raise ValueError(f"Invalid file extension '{file_extension}'") - matching_extensions = glob.glob(f'../tmp/*.{file_extension}') + matching_extensions = glob.glob(f"../tmp/*.{file_extension}") if not matching_extensions: logger.error(f"No files with the extension {file_extension} found") @@ -39,25 +40,27 @@ class Command(BaseCommand): filename = os.path.basename(src_file_path) do_command = True exit_status: int - - desired_file_path = f'{directory}/{filename}' + + desired_file_path = f"{directory}/{filename}" if os.path.exists(desired_file_path): - replace = input(f'{desired_file_path} already exists. Do you want to replace it? (y/n) ') - if replace.lower() != 'y': + # For linter + prompt = " Do you want to replace it? (y/n) " + replace = input(f"{desired_file_path} already exists. {prompt}") + if replace.lower() != "y": do_command = False - + if do_command: copy_from = f"../tmp/{filename}" self.cat(copy_from, desired_file_path) - exit_status = os.system(f'cat ../tmp/{filename} > {desired_file_path}') + exit_status = os.system(f"cat ../tmp/{filename} > {desired_file_path}") if exit_status == 0: logger.info(f"Successfully copied {filename}") else: logger.error(f"Failed to copy {filename}") - + def cat(self, copy_from, copy_to): - """Runs the cat command to + """Runs the cat command to copy_from a location to copy_to a location""" - exit_status = os.system(f'cat {copy_from} > {copy_to}') - return exit_status \ No newline at end of file + exit_status = os.system(f"cat {copy_from} > {copy_to}") + return exit_status diff --git a/src/registrar/management/commands/full_domain_migrations.py b/src/registrar/management/commands/full_domain_migrations.py index e2bee9da4..9144675ba 100644 --- a/src/registrar/management/commands/full_domain_migrations.py +++ b/src/registrar/management/commands/full_domain_migrations.py @@ -10,10 +10,6 @@ import argparse import sys import os -from django.test import Client - -from django_fsm import TransitionNotAllowed # type: ignore - from django.core.management import BaseCommand from registrar.models import ( @@ -21,19 +17,26 @@ from registrar.models import ( DomainInformation, DomainInvitation, TransitionDomain, - User, ) from registrar.management.commands.utility.terminal_helper import ( TerminalColors, - TerminalHelper + TerminalHelper, ) logger = logging.getLogger(__name__) + class Command(BaseCommand): help = """ """ + # Specifies which files the loaderFilenames commands should target + default_filenames = [ + "escrow_domain_contacts.daily.gov.GOV.txt", + "escrow_contacts.daily.gov.GOV.txt", + "escrow_domain_statuses.daily.gov.GOV.txt", + ] + def add_arguments(self, parser): """ OPTIONAL ARGUMENTS: @@ -51,8 +54,8 @@ class Command(BaseCommand): > --loaderDirectory /app/tmp --loaderFilenames - The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: + The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: EXAMPLE USAGE: > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt where... @@ -74,38 +77,49 @@ class Command(BaseCommand): purposes --resetTable - Used by the loaders to trigger a prompt for deleting all table entries. + Used by the loaders to trigger a prompt for deleting all table entries. Useful for testing purposes, but USE WITH CAUTION - """ + """ # noqa - line length - parser.add_argument("--runLoaders", + parser.add_argument( + "--runLoaders", help="Runs all scripts (in sequence) for transition domain migrations", - action=argparse.BooleanOptionalAction) - - parser.add_argument("--triggerLogins", + action=argparse.BooleanOptionalAction, + ) + + parser.add_argument( + "--triggerLogins", help="Simulates a user login for each user in domain invitation", - action=argparse.BooleanOptionalAction) + action=argparse.BooleanOptionalAction, + ) # The following file arguments have default values for running in the sandbox + # For linter + script = "load_transition_domain migration script" parser.add_argument( "--loaderDirectory", - default="migrationData", - help="The location of the files used for load_transition_domain migration script" + default="migrationdata", + help=f"The location of the files used for the {script}", ) parser.add_argument( "--loaderFilenames", - default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", - help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: + # For linter. + # Join all of the default filenames with a space. + # The resulting list will look like this: file1.txt file2.txt file3.txt + default=" ".join(self.default_filenames), + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""" + - domain_statuses_filename is the Data file with domain status information""", # noqa - linter length ) - parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") + parser.add_argument( + "--sep", default="|", help="Delimiter character for the loader files" + ) parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -119,22 +133,20 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - - def compare_tables(self, debug_on: bool): - """Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. - + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Produces the following report (printed to the terminal): #1 - Print any domains that exist in the transition_domain table but not in their corresponding domain, domain information or domain invitation tables. #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are + #3- Check for duplicate entries in domain or + domain_information tables and print which are duplicates and in which tables - """ - + """ + logger.info( f"""{TerminalColors.OKCYAN} ============= BEGINNING ANALYSIS =============== @@ -142,13 +154,13 @@ class Command(BaseCommand): """ ) - #TODO: would filteredRelation be faster? + # TODO: would filteredRelation be faster? missing_domains = [] duplicate_domains = [] missing_domain_informations = [] missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all():# DEBUG: + for transition_domain in TransitionDomain.objects.all(): # DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username @@ -160,22 +172,38 @@ class Command(BaseCommand): # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + matching_domain_informations = DomainInformation.objects.filter( + domain__name=transition_domain_name + ) # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), - domain__name=transition_domain_name) + matching_domain_invitations = DomainInvitation.objects.filter( + email=transition_domain_email.lower(), + domain__name=transition_domain_name, + ) if len(matching_domains) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""", # noqa - line length + ) missing_domains.append(transition_domain_name) elif len(matching_domains) > 1: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""", # noqa - line length + ) duplicate_domains.append(transition_domain_name) if len(matching_domain_informations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""", # noqa - line length + ) missing_domain_informations.append(transition_domain_name) if len(matching_domain_invitations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""", # noqa - line length + ) missing_domain_invites.append(transition_domain_name) total_missing_domains = len(missing_domains) @@ -184,10 +212,16 @@ class Command(BaseCommand): total_missing_domain_invitations = len(missing_domain_invites) missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) - duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) - missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) - missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) - + duplicate_domains_as_string = "{}".format( + ", ".join(map(str, duplicate_domains)) + ) + missing_domain_informations_as_string = "{}".format( + ", ".join(map(str, missing_domain_informations)) + ) + missing_domain_invites_as_string = "{}".format( + ", ".join(map(str, missing_domain_invites)) + ) + logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED ANALYSIS =============== @@ -208,9 +242,9 @@ class Command(BaseCommand): (These are transition domains which have no entires in the Domain Invitation Table) {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} {TerminalColors.ENDC} - """ + """ # noqa - line length ) - + def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: """Prompts the user to inspect the given terminal command string and asks if they wish to execute it. If the user responds (y), @@ -236,35 +270,39 @@ class Command(BaseCommand): # Otherwise, exit this subroutine. if not proceed_execution: sys.exit() - + self.execute_command(command_string) return True - - def execute_command(self, command_string:str): + + def execute_command(self, command_string: str): """Executes the given command string""" - logger.info(f"""{TerminalColors.OKCYAN} + logger.info( + f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== - {TerminalColors.ENDC}""") + {TerminalColors.ENDC}""" + ) os.system(f"{command_string}") - - def run_load_transition_domain_script(self, - file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, - sep: str, - reset_table: bool, - debug_on: bool, - prompts_enabled: bool, - debug_max_entries_to_parse: int): + + def run_load_transition_domain_script( + self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int, + ): """Runs the load_transition_domain script""" # Create the command string command_string = "./manage.py load_transition_domain " - command_string += file_location+domain_contacts_filename + " " - command_string += file_location+contacts_filename + " " - command_string += file_location+domain_statuses_filename + " " + command_string += file_location + domain_contacts_filename + " " + command_string += file_location + contacts_filename + " " + command_string += file_location + domain_statuses_filename + " " if sep is not None and sep != "|": command_string += f"--sep {sep} " if reset_table: @@ -276,12 +314,13 @@ class Command(BaseCommand): # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running load_transition_domain script") + self.prompt_for_execution( + command_string, "Running load_transition_domain script" + ) return self.execute_command(command_string) - - - def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): + + def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" # Create the command string command_string = "./manage.py transfer_transition_domains_to_domains " @@ -289,43 +328,42 @@ class Command(BaseCommand): command_string += "--debug " # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + self.prompt_for_execution( + command_string, "Running transfer_transition_domains_to_domains script" + ) return self.execute_command(command_string) - def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" # Create the command string... command_string = "./manage.py send_domain_invitations -s" # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running send_domain_invitations script") + self.prompt_for_execution( + command_string, "Running send_domain_invitations script" + ) return self.execute_command(command_string) + def run_migration_scripts(self, prompts_enabled: bool, options): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" - def run_migration_scripts(self, - prompts_enabled: bool, - options): - """Runs the following migration scripts (in order): - 1 - imports for trans domains - 2 - transfer to domain & domain invitation""" - # Get arguments sep = options.get("sep") reset_table = options.get("resetTable") debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) + debug_max_entries_to_parse = int(options.get("limitParse")) # Grab filepath information from the arguments - file_location = options.get("loaderDirectory")+"/" + file_location = options.get("loaderDirectory") + "/" filenames = options.get("loaderFilenames").split() if len(filenames) < 3: filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" + logger.info( + f""" {TerminalColors.FAIL} --loaderFilenames expected 3 filenames to follow it, but only {len(filenames)} were given: @@ -334,7 +372,8 @@ class Command(BaseCommand): PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN ============= TERMINATING ============= {TerminalColors.ENDC} - """) + """ + ) sys.exit() domain_contacts_filename = filenames[0] contacts_filename = filenames[1] @@ -362,45 +401,49 @@ class Command(BaseCommand): ) # If the user rejected the filepath information - # as incorrect, prompt the user to provide + # as incorrect, prompt the user to provide # correct file inputs in their original command # prompt and exit this subroutine if not files_are_correct: - logger.info(f""" + logger.info( + f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: EXAMPLE: docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - """) + """ # noqa - line length + ) return - - # Proceed executing the migration scripts - self.run_load_transition_domain_script(file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse) - self.run_transfer_script(debug_on, prompts_enabled) + # Proceed executing the migration scripts + self.run_load_transition_domain_script( + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse, + ) + self.run_transfer_script(debug_on, prompts_enabled) def simulate_user_logins(self, debug_on): """Simulates logins for users (this will add Domain Information objects to our tables)""" - logger.info(f"" - f"{TerminalColors.OKCYAN}" - f"================== SIMULATING LOGINS ==================" - f"{TerminalColors.ENDC}") - + logger.info( + f"" + f"{TerminalColors.OKCYAN}" + f"================== SIMULATING LOGINS ==================" + f"{TerminalColors.ENDC}" + ) + command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins" - - + # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff # #DEBUG: # TerminalHelper.print_conditional(debug_on, @@ -419,7 +462,6 @@ class Command(BaseCommand): # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") # user.delete() - def handle( self, **options, @@ -430,7 +472,7 @@ class Command(BaseCommand): 2 - simulate logins 3 - send domain invitations (Emails should be sent to the appropriate users note that all moved domains should now be accessible - on django admin for an analyst) + on django admin for an analyst) 4 - analyze the data for transition domains and generate a report """ @@ -442,34 +484,34 @@ class Command(BaseCommand): # the terminal so the user knows what is # enabled. debug_on = options.get("debug") - prompts_enabled = debug_on #TODO: add as argument? + prompts_enabled = debug_on # TODO: add as argument? run_loaders_enabled = options.get("runLoaders") simulate_user_login_enabled = options.get("triggerLogins") TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} + debug_on, + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. {TerminalColors.ENDC} - """ - ) + """, + ) TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} ----------RUNNING LOADERS ON---------- All migration scripts will be run before analyzing the data. {TerminalColors.ENDC} - """ - ) + """, + ) TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} ----------TRIGGER LOGINS ON---------- Will be simulating user logins {TerminalColors.ENDC} - """ - ) + """, + ) # If a user decides to run all migration # scripts, they may or may not wish to # proceed with analysis of the data depending @@ -486,7 +528,7 @@ class Command(BaseCommand): if run_loaders_enabled: self.run_migration_scripts(options, prompts_enabled) prompt_continuation_of_analysis = True - + # STEP 2 -- SIMULATE LOGINS # Simulate user login for each user in domain # invitation if specified by user OR if running @@ -500,7 +542,7 @@ class Command(BaseCommand): if run_loaders_enabled: if prompts_enabled: simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} + f"""{TerminalColors.FAIL} Proceed with simulating user logins? {TerminalColors.ENDC}""" ) @@ -508,14 +550,14 @@ class Command(BaseCommand): return self.simulate_user_logins(debug_on) prompt_continuation_of_analysis = True - + # STEP 3 -- SEND INVITES if prompts_enabled: proceed_with_sending_invites = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with sending user invites? {TerminalColors.ENDC}""" - ) + ) if not proceed_with_sending_invites: return self.run_send_invites_script(debug_on) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index ee1e8c4ea..b2eefae54 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -12,7 +12,7 @@ from registrar.models import TransitionDomain from registrar.management.commands.utility.terminal_helper import ( TerminalColors, - TerminalHelper + TerminalHelper, ) logger = logging.getLogger(__name__) diff --git a/src/registrar/management/commands/test_domain_migration.py b/src/registrar/management/commands/test_domain_migration.py deleted file mode 100644 index 781de38e0..000000000 --- a/src/registrar/management/commands/test_domain_migration.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Data migration: - 1 - generates a report of data integrity across all - transition domain related tables - 2 - allows users to run all migration scripts for - transition domain data -""" - -import logging -import argparse -import sys -import os - -from django.test import Client - -from django_fsm import TransitionNotAllowed # type: ignore - -from django.core.management import BaseCommand - -from registrar.models import ( - Domain, - DomainInformation, - DomainInvitation, - TransitionDomain, - User, -) - -from registrar.management.commands.utility.terminal_helper import ( - TerminalColors, - TerminalHelper -) - -logger = logging.getLogger(__name__) - -class Command(BaseCommand): - help = """ """ - - def add_arguments(self, parser): - """ - OPTIONAL ARGUMENTS: - --runLoaders - A boolean (default to true), which triggers running - all scripts (in sequence) for transition domain migrations - - --triggerLogins - A boolean (default to true), which triggers running - simulations of user logins for each user in domain invitation - - --loaderDirectory - The location of the files used for load_transition_domain migration script - EXAMPLE USAGE: - > --loaderDirectory /app/tmp - - --loaderFilenames - The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: - EXAMPLE USAGE: - > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt - where... - - domain_contacts_filename is the Data file with domain contact information - - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information - - --sep - Delimiter for the loaders to correctly parse the given text files. - (usually this can remain at default value of |) - - --debug - A boolean (default to true), which activates additional print statements - - --limitParse - Used by the loaders (load_transition_domain) to set the limit for the - number of data entries to insert. Set to 0 (or just don't use this - argument) to parse every entry. This was provided primarily for testing - purposes - - --resetTable - Used by the loaders to trigger a prompt for deleting all table entries. - Useful for testing purposes, but USE WITH CAUTION - """ - - parser.add_argument("--runLoaders", - help="Runs all scripts (in sequence) for transition domain migrations", - action=argparse.BooleanOptionalAction) - - parser.add_argument("--triggerLogins", - help="Simulates a user login for each user in domain invitation", - action=argparse.BooleanOptionalAction) - - # The following file arguments have default values for running in the sandbox - parser.add_argument( - "--loaderDirectory", - default="migrationData", - help="The location of the files used for load_transition_domain migration script" - ) - parser.add_argument( - "--loaderFilenames", - default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", - help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: - domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt - - where... - - domain_contacts_filename is the Data file with domain contact information - - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""" - ) - - parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") - - parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - - parser.add_argument( - "--limitParse", default=0, help="Sets max number of entries to load" - ) - - parser.add_argument( - "--resetTable", - help="Deletes all data in the TransitionDomain table", - action=argparse.BooleanOptionalAction, - ) - - - - def compare_tables(self, debug_on: bool): - """Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. - - Produces the following report (printed to the terminal): - #1 - Print any domains that exist in the transition_domain table - but not in their corresponding domain, domain information or - domain invitation tables. - #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are - duplicates and in which tables - """ - - logger.info( - f"""{TerminalColors.OKCYAN} - ============= BEGINNING ANALYSIS =============== - {TerminalColors.ENDC} - """ - ) - - #TODO: would filteredRelation be faster? - - missing_domains = [] - duplicate_domains = [] - missing_domain_informations = [] - missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all():# DEBUG: - transition_domain_name = transition_domain.domain_name - transition_domain_email = transition_domain.username - - TerminalHelper.print_conditional( - debug_on, - f"{TerminalColors.OKCYAN}Checking: {transition_domain_name} {TerminalColors.ENDC}", # noqa - ) - - # Check Domain table - matching_domains = Domain.objects.filter(name=transition_domain_name) - # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) - # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), - domain__name=transition_domain_name) - - if len(matching_domains) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") - missing_domains.append(transition_domain_name) - elif len(matching_domains) > 1: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") - duplicate_domains.append(transition_domain_name) - if len(matching_domain_informations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") - missing_domain_informations.append(transition_domain_name) - if len(matching_domain_invitations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") - missing_domain_invites.append(transition_domain_name) - - total_missing_domains = len(missing_domains) - total_duplicate_domains = len(duplicate_domains) - total_missing_domain_informations = len(missing_domain_informations) - total_missing_domain_invitations = len(missing_domain_invites) - - missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) - duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) - missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) - missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) - - logger.info( - f"""{TerminalColors.OKGREEN} - ============= FINISHED ANALYSIS =============== - - {total_missing_domains} Missing Domains: - (These are transition domains that are missing from the Domain Table) - {TerminalColors.YELLOW}{missing_domains_as_string}{TerminalColors.OKGREEN} - - {total_duplicate_domains} Duplicate Domains: - (These are transition domains which have duplicate entries in the Domain Table) - {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} - - {total_missing_domain_informations} Domain Information Entries missing: - (These are transition domains which have no entries in the Domain Information Table) - {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} - - {total_missing_domain_invitations} Domain Invitations missing: - (These are transition domains which have no entires in the Domain Invitation Table) - {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} - {TerminalColors.ENDC} - """ - ) - - def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: - """Prompts the user to inspect the given terminal command string - and asks if they wish to execute it. If the user responds (y), - execute the command""" - - # Allow the user to inspect the command string - # and ask if they wish to proceed - proceed_execution = TerminalHelper.query_yes_no( - f"""{TerminalColors.OKCYAN} - ===================================================== - {prompt_title} - ===================================================== - *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** - - {command_string} - {TerminalColors.FAIL} - Proceed? (Y = proceed, N = skip) - {TerminalColors.ENDC}""" - ) - - # If the user decided to proceed executing the command, - # run the command for loading transition domains. - # Otherwise, exit this subroutine. - if not proceed_execution: - sys.exit() - - self.execute_command(command_string) - - return True - - def execute_command(self, command_string:str): - """Executes the given command string""" - - logger.info(f"""{TerminalColors.OKCYAN} - ==== EXECUTING... ==== - {TerminalColors.ENDC}""") - os.system(f"{command_string}") - - def run_load_transition_domain_script(self, - file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, - sep: str, - reset_table: bool, - debug_on: bool, - prompts_enabled: bool, - debug_max_entries_to_parse: int): - """Runs the load_transition_domain script""" - # Create the command string - command_string = "./manage.py load_transition_domain " - command_string += file_location+domain_contacts_filename + " " - command_string += file_location+contacts_filename + " " - command_string += file_location+domain_statuses_filename + " " - if sep is not None and sep != "|": - command_string += f"--sep {sep} " - if reset_table: - command_string += "--resetTable " - if debug_on: - command_string += "--debug " - if debug_max_entries_to_parse > 0: - command_string += f"--limitParse {debug_max_entries_to_parse} " - - # Execute the command string - if prompts_enabled: - self.prompt_for_execution(command_string, "Running load_transition_domain script") - return - self.execute_command(command_string) - - - def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): - """Runs the transfer_transition_domains_to_domains script""" - # Create the command string - command_string = "./manage.py transfer_transition_domains_to_domains " - if debug_on: - command_string += "--debug " - # Execute the command string - if prompts_enabled: - self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") - return - self.execute_command(command_string) - - - def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): - """Runs the send_domain_invitations script""" - # Create the command string... - command_string = "./manage.py send_domain_invitations -s" - # Execute the command string - if prompts_enabled: - self.prompt_for_execution(command_string, "Running send_domain_invitations script") - return - self.execute_command(command_string) - - - def run_migration_scripts(self, - prompts_enabled: bool, - options): - """Runs the following migration scripts (in order): - 1 - imports for trans domains - 2 - transfer to domain & domain invitation""" - - # Get arguments - sep = options.get("sep") - reset_table = options.get("resetTable") - debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) - - # Grab filepath information from the arguments - file_location = options.get("loaderDirectory")+"/" - filenames = options.get("loaderFilenames").split() - if len(filenames) < 3: - filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" - {TerminalColors.FAIL} - --loaderFilenames expected 3 filenames to follow it, - but only {len(filenames)} were given: - {filenames_as_string} - - PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN - ============= TERMINATING ============= - {TerminalColors.ENDC} - """) - sys.exit() - domain_contacts_filename = filenames[0] - contacts_filename = filenames[1] - domain_statuses_filename = filenames[2] - - if prompts_enabled: - # Allow the user to inspect the filepath - # data given in the arguments, and prompt - # the user to verify this info before proceeding - files_are_correct = TerminalHelper.query_yes_no( - f""" - {TerminalColors.OKCYAN} - *** IMPORTANT: VERIFY THE FOLLOWING *** - - The migration scripts are looking in directory.... - {file_location} - - ....for the following files: - - domain contacts: {domain_contacts_filename} - - contacts: {contacts_filename} - - domain statuses: {domain_statuses_filename} - - {TerminalColors.FAIL} - Does this look correct?{TerminalColors.ENDC}""" - ) - - # If the user rejected the filepath information - # as incorrect, prompt the user to provide - # correct file inputs in their original command - # prompt and exit this subroutine - if not files_are_correct: - logger.info(f""" - {TerminalColors.YELLOW} - PLEASE Re-Run the script with the correct file location and filenames: - - EXAMPLE: - docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - - """) - return - - # Proceed executing the migration scripts - self.run_load_transition_domain_script(file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse) - self.run_transfer_script(debug_on, prompts_enabled) - - - def simulate_user_logins(self, debug_on): - """Simulates logins for users (this will add - Domain Information objects to our tables)""" - - logger.info(f"" - f"{TerminalColors.OKCYAN}" - f"================== SIMULATING LOGINS ==================" - f"{TerminalColors.ENDC}") - - - - # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff - # #DEBUG: - # TerminalHelper.print_conditional(debug_on, - # f"{TerminalColors.OKCYAN}" - # f"Processing invite: {invite}" - # f"{TerminalColors.ENDC}") - # # get a user with this email address - # user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) - # #DEBUG: - # TerminalHelper.print_conditional(user_created, - # f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") - # TerminalHelper.print_conditional(debug_on, - # f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") - # user.first_login() - # if user_created: - # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") - # user.delete() - - - def handle( - self, - **options, - ): - """ - Does the following; - 1 - run loader scripts - 2 - simulate logins - 3 - send domain invitations (Emails should be sent to the appropriate users - note that all moved domains should now be accessible - on django admin for an analyst) - 4 - analyze the data for transition domains - and generate a report - """ - - # SETUP - # Grab all arguments relevant to - # orchestrating which parts of this script - # should execute. Print some indicators to - # the terminal so the user knows what is - # enabled. - debug_on = options.get("debug") - prompts_enabled = debug_on #TODO: add as argument? - run_loaders_enabled = options.get("runLoaders") - simulate_user_login_enabled = options.get("triggerLogins") - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} - ----------DEBUG MODE ON---------- - Detailed print statements activated. - {TerminalColors.ENDC} - """ - ) - TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} - ----------RUNNING LOADERS ON---------- - All migration scripts will be run before - analyzing the data. - {TerminalColors.ENDC} - """ - ) - TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} - ----------TRIGGER LOGINS ON---------- - Will be simulating user logins - {TerminalColors.ENDC} - """ - ) - # If a user decides to run all migration - # scripts, they may or may not wish to - # proceed with analysis of the data depending - # on the results of the migration. - # Provide a breakpoint for them to decide - # whether to continue or not. - # The same will happen if simulating user - # logins (to allow users to run only that - # portion of the script if desired) - prompt_continuation_of_analysis = False - - # STEP 1 -- RUN LOADERS - # Run migration scripts if specified by user - if run_loaders_enabled: - self.run_migration_scripts(options, prompts_enabled) - prompt_continuation_of_analysis = True - - # STEP 2 -- SIMULATE LOGINS - # Simulate user login for each user in domain - # invitation if specified by user OR if running - # migration scripts. - # (NOTE: Although users can choose to run login - # simulations separately (for testing purposes), - # if we are running all migration scripts, we should - # automatically execute this as the final step - # to ensure Domain Information objects get added - # to the database.) - if run_loaders_enabled: - if prompts_enabled: - simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with simulating user logins? - {TerminalColors.ENDC}""" - ) - if not simulate_user_login_enabled: - return - self.simulate_user_logins(debug_on) - prompt_continuation_of_analysis = True - - # STEP 3 -- SEND INVITES - if prompts_enabled: - proceed_with_sending_invites = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with sending user invites? - {TerminalColors.ENDC}""" - ) - if not proceed_with_sending_invites: - return - self.run_send_invites_script(debug_on) - prompt_continuation_of_analysis = True - - # STEP 4 -- ANALYZE TABLES & GENERATE REPORT - # Analyze tables for corrupt data... - if prompt_continuation_of_analysis & prompts_enabled: - # ^ (only prompt if we ran steps 1 and/or 2) - analyze_tables = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with table analysis? - {TerminalColors.ENDC}""" - ) - if not analyze_tables: - return - self.compare_tables(debug_on) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 9443f085f..056b8d447 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -12,7 +12,7 @@ from registrar.models import DomainInvitation from registrar.management.commands.utility.terminal_helper import ( TerminalColors, - TerminalHelper + TerminalHelper, ) logger = logging.getLogger(__name__) @@ -57,7 +57,6 @@ class Command(BaseCommand): """, ) - def update_domain_status( self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool ) -> bool: diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 3f647f0b8..99064346d 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -2,6 +2,7 @@ import logging logger = logging.getLogger(__name__) + class TerminalColors: """Colors for terminal outputs (makes reading the logs WAY easier)""" @@ -18,8 +19,8 @@ class TerminalColors: UNDERLINE = "\033[4m" BackgroundLightYellow = "\033[103m" -class TerminalHelper: +class TerminalHelper: def query_yes_no(question: str, default="yes") -> bool: """Ask a yes/no question via raw_input() and return their answer. @@ -57,4 +58,4 @@ class TerminalHelper: terminal if print_condition is TRUE""" # DEBUG: if print_condition: - logger.info(print_statement) \ No newline at end of file + logger.info(print_statement) diff --git a/src/registrar/tests/test_transition_domain_migrations_with_logins.py b/src/registrar/tests/test_transition_domain_migrations_with_logins.py index da2c31877..4b723bc8a 100644 --- a/src/registrar/tests/test_transition_domain_migrations_with_logins.py +++ b/src/registrar/tests/test_transition_domain_migrations_with_logins.py @@ -1,26 +1,13 @@ from django.test import TestCase -from django.db.utils import IntegrityError -from unittest.mock import patch from registrar.models import ( - Contact, - DomainApplication, - DomainInformation, User, - Website, Domain, - DraftDomain, DomainInvitation, UserDomainRole, ) -import boto3_mocking # type: ignore -from .common import MockSESClient, less_console_noise, completed_application -from django_fsm import TransitionNotAllowed -boto3_mocking.clients.register_handler("sesv2", MockSESClient) - -@boto3_mocking.patching class TestLogins(TestCase): """Test the retrieval of invitations.""" From b62a9b4223f6c9704ca67cced9761936322292fb Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 25 Oct 2023 11:26:26 -0600 Subject: [PATCH 14/49] file renames Signed-off-by: CocoByte --- ...rations.py => master_domain_migrations.py} | 317 ++++++++---------- ...y => test_transition_domain_migrations.py} | 8 + 2 files changed, 146 insertions(+), 179 deletions(-) rename src/registrar/management/commands/{full_domain_migrations.py => master_domain_migrations.py} (72%) rename src/registrar/tests/{test_transition_domain_migrations_with_logins.py => test_transition_domain_migrations.py} (75%) diff --git a/src/registrar/management/commands/full_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py similarity index 72% rename from src/registrar/management/commands/full_domain_migrations.py rename to src/registrar/management/commands/master_domain_migrations.py index 9144675ba..a90be402b 100644 --- a/src/registrar/management/commands/full_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -10,6 +10,10 @@ import argparse import sys import os +from django.test import Client + +from django_fsm import TransitionNotAllowed # type: ignore + from django.core.management import BaseCommand from registrar.models import ( @@ -17,26 +21,19 @@ from registrar.models import ( DomainInformation, DomainInvitation, TransitionDomain, + User, ) from registrar.management.commands.utility.terminal_helper import ( TerminalColors, - TerminalHelper, + TerminalHelper ) logger = logging.getLogger(__name__) - class Command(BaseCommand): help = """ """ - # Specifies which files the loaderFilenames commands should target - default_filenames = [ - "escrow_domain_contacts.daily.gov.GOV.txt", - "escrow_contacts.daily.gov.GOV.txt", - "escrow_domain_statuses.daily.gov.GOV.txt", - ] - def add_arguments(self, parser): """ OPTIONAL ARGUMENTS: @@ -54,8 +51,8 @@ class Command(BaseCommand): > --loaderDirectory /app/tmp --loaderFilenames - The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: + The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: EXAMPLE USAGE: > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt where... @@ -77,49 +74,38 @@ class Command(BaseCommand): purposes --resetTable - Used by the loaders to trigger a prompt for deleting all table entries. + Used by the loaders to trigger a prompt for deleting all table entries. Useful for testing purposes, but USE WITH CAUTION - """ # noqa - line length + """ - parser.add_argument( - "--runLoaders", + parser.add_argument("--runLoaders", help="Runs all scripts (in sequence) for transition domain migrations", - action=argparse.BooleanOptionalAction, - ) - - parser.add_argument( - "--triggerLogins", + action=argparse.BooleanOptionalAction) + + parser.add_argument("--triggerLogins", help="Simulates a user login for each user in domain invitation", - action=argparse.BooleanOptionalAction, - ) + action=argparse.BooleanOptionalAction) # The following file arguments have default values for running in the sandbox - # For linter - script = "load_transition_domain migration script" parser.add_argument( "--loaderDirectory", - default="migrationdata", - help=f"The location of the files used for the {script}", + default="migrationData", + help="The location of the files used for load_transition_domain migration script" ) parser.add_argument( "--loaderFilenames", - # For linter. - # Join all of the default filenames with a space. - # The resulting list will look like this: file1.txt file2.txt file3.txt - default=" ".join(self.default_filenames), - help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: + default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by spaces: domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""", # noqa - linter length + - domain_statuses_filename is the Data file with domain status information""" ) - parser.add_argument( - "--sep", default="|", help="Delimiter character for the loader files" - ) + parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -133,20 +119,22 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - def compare_tables(self, debug_on: bool): - """Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. + + def compare_tables(self, debug_on: bool): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Produces the following report (printed to the terminal): #1 - Print any domains that exist in the transition_domain table but not in their corresponding domain, domain information or domain invitation tables. #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are + #3- Check for duplicate entries in domain or + domain_information tables and print which are duplicates and in which tables - """ - + """ + logger.info( f"""{TerminalColors.OKCYAN} ============= BEGINNING ANALYSIS =============== @@ -154,13 +142,13 @@ class Command(BaseCommand): """ ) - # TODO: would filteredRelation be faster? + #TODO: would filteredRelation be faster? missing_domains = [] duplicate_domains = [] missing_domain_informations = [] missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all(): # DEBUG: + for transition_domain in TransitionDomain.objects.all():# DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username @@ -172,38 +160,22 @@ class Command(BaseCommand): # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter( - domain__name=transition_domain_name - ) + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter( - email=transition_domain_email.lower(), - domain__name=transition_domain_name, - ) + matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), + domain__name=transition_domain_name) if len(matching_domains) == 0: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""", # noqa - line length - ) + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") missing_domains.append(transition_domain_name) elif len(matching_domains) > 1: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""", # noqa - line length - ) + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") duplicate_domains.append(transition_domain_name) if len(matching_domain_informations) == 0: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""", # noqa - line length - ) + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") missing_domain_informations.append(transition_domain_name) if len(matching_domain_invitations) == 0: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""", # noqa - line length - ) + TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") missing_domain_invites.append(transition_domain_name) total_missing_domains = len(missing_domains) @@ -212,16 +184,10 @@ class Command(BaseCommand): total_missing_domain_invitations = len(missing_domain_invites) missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) - duplicate_domains_as_string = "{}".format( - ", ".join(map(str, duplicate_domains)) - ) - missing_domain_informations_as_string = "{}".format( - ", ".join(map(str, missing_domain_informations)) - ) - missing_domain_invites_as_string = "{}".format( - ", ".join(map(str, missing_domain_invites)) - ) - + duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) + missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) + missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) + logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED ANALYSIS =============== @@ -242,9 +208,9 @@ class Command(BaseCommand): (These are transition domains which have no entires in the Domain Invitation Table) {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} {TerminalColors.ENDC} - """ # noqa - line length + """ ) - + def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: """Prompts the user to inspect the given terminal command string and asks if they wish to execute it. If the user responds (y), @@ -270,39 +236,35 @@ class Command(BaseCommand): # Otherwise, exit this subroutine. if not proceed_execution: sys.exit() - + self.execute_command(command_string) return True - - def execute_command(self, command_string: str): + + def execute_command(self, command_string:str): """Executes the given command string""" - logger.info( - f"""{TerminalColors.OKCYAN} + logger.info(f"""{TerminalColors.OKCYAN} ==== EXECUTING... ==== - {TerminalColors.ENDC}""" - ) + {TerminalColors.ENDC}""") os.system(f"{command_string}") - - def run_load_transition_domain_script( - self, - file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, - sep: str, - reset_table: bool, - debug_on: bool, - prompts_enabled: bool, - debug_max_entries_to_parse: int, - ): + + def run_load_transition_domain_script(self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int): """Runs the load_transition_domain script""" # Create the command string command_string = "./manage.py load_transition_domain " - command_string += file_location + domain_contacts_filename + " " - command_string += file_location + contacts_filename + " " - command_string += file_location + domain_statuses_filename + " " + command_string += file_location+domain_contacts_filename + " " + command_string += file_location+contacts_filename + " " + command_string += file_location+domain_statuses_filename + " " if sep is not None and sep != "|": command_string += f"--sep {sep} " if reset_table: @@ -314,13 +276,12 @@ class Command(BaseCommand): # Execute the command string if prompts_enabled: - self.prompt_for_execution( - command_string, "Running load_transition_domain script" - ) + self.prompt_for_execution(command_string, "Running load_transition_domain script") return self.execute_command(command_string) - - def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): + + + def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" # Create the command string command_string = "./manage.py transfer_transition_domains_to_domains " @@ -328,42 +289,43 @@ class Command(BaseCommand): command_string += "--debug " # Execute the command string if prompts_enabled: - self.prompt_for_execution( - command_string, "Running transfer_transition_domains_to_domains script" - ) + self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") return self.execute_command(command_string) + def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" # Create the command string... command_string = "./manage.py send_domain_invitations -s" # Execute the command string if prompts_enabled: - self.prompt_for_execution( - command_string, "Running send_domain_invitations script" - ) + self.prompt_for_execution(command_string, "Running send_domain_invitations script") return self.execute_command(command_string) - def run_migration_scripts(self, prompts_enabled: bool, options): - """Runs the following migration scripts (in order): - 1 - imports for trans domains - 2 - transfer to domain & domain invitation""" + def run_migration_scripts(self, + prompts_enabled: bool, + options): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" + # Get arguments sep = options.get("sep") reset_table = options.get("resetTable") debug_on = options.get("debug") - debug_max_entries_to_parse = int(options.get("limitParse")) + debug_max_entries_to_parse = int( + options.get("limitParse") + ) # Grab filepath information from the arguments - file_location = options.get("loaderDirectory") + "/" - filenames = options.get("loaderFilenames").split() + file_location = options.get("loaderDirectory")+"/" + filenames = options.get("loaderFilenames").split(",") if len(filenames) < 3: filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info( - f""" + logger.info(f""" {TerminalColors.FAIL} --loaderFilenames expected 3 filenames to follow it, but only {len(filenames)} were given: @@ -372,8 +334,7 @@ class Command(BaseCommand): PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN ============= TERMINATING ============= {TerminalColors.ENDC} - """ - ) + """) sys.exit() domain_contacts_filename = filenames[0] contacts_filename = filenames[1] @@ -401,49 +362,45 @@ class Command(BaseCommand): ) # If the user rejected the filepath information - # as incorrect, prompt the user to provide + # as incorrect, prompt the user to provide # correct file inputs in their original command # prompt and exit this subroutine if not files_are_correct: - logger.info( - f""" + logger.info(f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: EXAMPLE: docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - """ # noqa - line length - ) + """) return - + # Proceed executing the migration scripts - self.run_load_transition_domain_script( - file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse, - ) + self.run_load_transition_domain_script(file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse) self.run_transfer_script(debug_on, prompts_enabled) + def simulate_user_logins(self, debug_on): """Simulates logins for users (this will add Domain Information objects to our tables)""" - logger.info( - f"" - f"{TerminalColors.OKCYAN}" - f"================== SIMULATING LOGINS ==================" - f"{TerminalColors.ENDC}" - ) - - command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins" + logger.info(f"" + f"{TerminalColors.OKCYAN}" + f"================== SIMULATING LOGINS ==================" + f"{TerminalColors.ENDC}") + + # command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins" + # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff # #DEBUG: # TerminalHelper.print_conditional(debug_on, @@ -462,6 +419,7 @@ class Command(BaseCommand): # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") # user.delete() + def handle( self, **options, @@ -472,7 +430,7 @@ class Command(BaseCommand): 2 - simulate logins 3 - send domain invitations (Emails should be sent to the appropriate users note that all moved domains should now be accessible - on django admin for an analyst) + on django admin for an analyst) 4 - analyze the data for transition domains and generate a report """ @@ -484,34 +442,34 @@ class Command(BaseCommand): # the terminal so the user knows what is # enabled. debug_on = options.get("debug") - prompts_enabled = debug_on # TODO: add as argument? + prompts_enabled = debug_on #TODO: add as argument? run_loaders_enabled = options.get("runLoaders") simulate_user_login_enabled = options.get("triggerLogins") TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} + debug_on, + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. {TerminalColors.ENDC} - """, - ) + """ + ) TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} ----------RUNNING LOADERS ON---------- All migration scripts will be run before analyzing the data. {TerminalColors.ENDC} - """, - ) + """ + ) TerminalHelper.print_conditional( - run_loaders_enabled, - f"""{TerminalColors.OKCYAN} + run_loaders_enabled, + f"""{TerminalColors.OKCYAN} ----------TRIGGER LOGINS ON---------- Will be simulating user logins {TerminalColors.ENDC} - """, - ) + """ + ) # If a user decides to run all migration # scripts, they may or may not wish to # proceed with analysis of the data depending @@ -526,9 +484,9 @@ class Command(BaseCommand): # STEP 1 -- RUN LOADERS # Run migration scripts if specified by user if run_loaders_enabled: - self.run_migration_scripts(options, prompts_enabled) + self.run_migration_scripts(prompts_enabled, options) prompt_continuation_of_analysis = True - + # STEP 2 -- SIMULATE LOGINS # Simulate user login for each user in domain # invitation if specified by user OR if running @@ -539,25 +497,26 @@ class Command(BaseCommand): # automatically execute this as the final step # to ensure Domain Information objects get added # to the database.) - if run_loaders_enabled: - if prompts_enabled: - simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with simulating user logins? - {TerminalColors.ENDC}""" - ) - if not simulate_user_login_enabled: - return - self.simulate_user_logins(debug_on) - prompt_continuation_of_analysis = True - + + # if run_loaders_enabled: + # if prompts_enabled: + # simulate_user_login_enabled = TerminalHelper.query_yes_no( + # f"""{TerminalColors.FAIL} + # Proceed with simulating user logins? + # {TerminalColors.ENDC}""" + # ) + # if not simulate_user_login_enabled: + # return + # self.simulate_user_logins(debug_on) + # prompt_continuation_of_analysis = True + # STEP 3 -- SEND INVITES if prompts_enabled: proceed_with_sending_invites = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with sending user invites? {TerminalColors.ENDC}""" - ) + ) if not proceed_with_sending_invites: return self.run_send_invites_script(debug_on) diff --git a/src/registrar/tests/test_transition_domain_migrations_with_logins.py b/src/registrar/tests/test_transition_domain_migrations.py similarity index 75% rename from src/registrar/tests/test_transition_domain_migrations_with_logins.py rename to src/registrar/tests/test_transition_domain_migrations.py index 4b723bc8a..b9a9d2c38 100644 --- a/src/registrar/tests/test_transition_domain_migrations_with_logins.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -23,6 +23,14 @@ class TestLogins(TestCase): # clean out the roles each time UserDomainRole.objects.all().delete() + def test_migration_functions(self): + """ Run the master migration script using local test data """ + + """ (analyze the tables just like the migration script does, but add assert statements) """ + #TODO: finish me! + self.assertTrue(True) + + def test_user_logins(self): """A new user's first_login callback retrieves their invitations.""" self.user.first_login() From 1b5d7a624890cb09ba10af90f1cc0e42f04c5c9c Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 25 Oct 2023 18:38:39 -0600 Subject: [PATCH 15/49] Auto stash before merge of "nl/981-test-domain-migration-script" and "origin/nl/981-test-domain-migration-script" Unit test experiments... --- .../commands/load_transition_domain.py | 5 +- .../commands/master_domain_migrations.py | 224 ++++++++---------- .../transfer_transition_domains_to_domains.py | 4 +- .../commands/utility/terminal_helper.py | 47 ++++ .../tests/data/test_domain_contacts.txt | 14 +- .../tests/data/test_domain_statuses.txt | 4 +- .../test_transition_domain_migrations.py | 159 +++++++++++-- 7 files changed, 302 insertions(+), 155 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index b2eefae54..3dabd68d9 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -101,11 +101,12 @@ class Command(BaseCommand): return domain_status_dictionary def get_user_emails_dict( - self, contacts_filename: str, sep + self, + contacts_filename: str, sep ) -> defaultdict[str, str]: """Creates mapping of userId -> emails""" user_emails_dictionary = defaultdict(str) - logger.info("Reading domain-contacts data file %s", contacts_filename) + logger.info("Reading contacts data file %s", contacts_filename) with open(contacts_filename, "r") as contacts_file: for row in csv.reader(contacts_file, delimiter=sep): user_id = row[0] diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index a90be402b..a7cd0dcc2 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -37,67 +37,71 @@ class Command(BaseCommand): def add_arguments(self, parser): """ OPTIONAL ARGUMENTS: - --runLoaders + --runMigrations A boolean (default to true), which triggers running all scripts (in sequence) for transition domain migrations - --triggerLogins - A boolean (default to true), which triggers running - simulations of user logins for each user in domain invitation - - --loaderDirectory + --migrationDirectory The location of the files used for load_transition_domain migration script EXAMPLE USAGE: - > --loaderDirectory /app/tmp + > --migrationDirectory /app/tmp - --loaderFilenames + --migrationFilenames The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: + Must appear IN ORDER and comma-delimiteds: EXAMPLE USAGE: - > --loaderFilenames domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + > --migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - domain_statuses_filename is the Data file with domain status information --sep - Delimiter for the loaders to correctly parse the given text files. + Delimiter for the migration scripts to correctly parse the given text files. (usually this can remain at default value of |) --debug A boolean (default to true), which activates additional print statements + --prompt + A boolean (default to true), which activates terminal prompts + that allows the user to step through each portion of this + script. + --limitParse - Used by the loaders (load_transition_domain) to set the limit for the + Used by the migration scripts (load_transition_domain) to set the limit for the number of data entries to insert. Set to 0 (or just don't use this argument) to parse every entry. This was provided primarily for testing purposes --resetTable - Used by the loaders to trigger a prompt for deleting all table entries. + Used by the migration scripts to trigger a prompt for deleting all table entries. Useful for testing purposes, but USE WITH CAUTION """ - parser.add_argument("--runLoaders", + parser.add_argument("--runMigrations", help="Runs all scripts (in sequence) for transition domain migrations", action=argparse.BooleanOptionalAction) + # --triggerLogins + # A boolean (default to true), which triggers running + # simulations of user logins for each user in domain invitation parser.add_argument("--triggerLogins", help="Simulates a user login for each user in domain invitation", action=argparse.BooleanOptionalAction) - + # The following file arguments have default values for running in the sandbox parser.add_argument( - "--loaderDirectory", + "--migrationDirectory", default="migrationData", help="The location of the files used for load_transition_domain migration script" ) parser.add_argument( - "--loaderFilenames", + "--migrationFilenames", default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by spaces: - domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt + Must appear IN ORDER and separated by commas: + domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt where... - domain_contacts_filename is the Data file with domain contact information @@ -105,10 +109,12 @@ class Command(BaseCommand): - domain_statuses_filename is the Data file with domain status information""" ) - parser.add_argument("--sep", default="|", help="Delimiter character for the loader files") + parser.add_argument("--sep", default="|", help="Delimiter character for the migration files") parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) + parser.add_argument( "--limitParse", default=0, help="Sets max number of entries to load" ) @@ -119,8 +125,6 @@ class Command(BaseCommand): action=argparse.BooleanOptionalAction, ) - - def compare_tables(self, debug_on: bool): """Does a diff between the transition_domain and the following tables: domain, domain_information and the domain_invitation. @@ -211,44 +215,6 @@ class Command(BaseCommand): """ ) - def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: - """Prompts the user to inspect the given terminal command string - and asks if they wish to execute it. If the user responds (y), - execute the command""" - - # Allow the user to inspect the command string - # and ask if they wish to proceed - proceed_execution = TerminalHelper.query_yes_no( - f"""{TerminalColors.OKCYAN} - ===================================================== - {prompt_title} - ===================================================== - *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** - - {command_string} - {TerminalColors.FAIL} - Proceed? (Y = proceed, N = skip) - {TerminalColors.ENDC}""" - ) - - # If the user decided to proceed executing the command, - # run the command for loading transition domains. - # Otherwise, exit this subroutine. - if not proceed_execution: - sys.exit() - - self.execute_command(command_string) - - return True - - def execute_command(self, command_string:str): - """Executes the given command string""" - - logger.info(f"""{TerminalColors.OKCYAN} - ==== EXECUTING... ==== - {TerminalColors.ENDC}""") - os.system(f"{command_string}") - def run_load_transition_domain_script(self, file_location: str, domain_contacts_filename: str, @@ -276,9 +242,10 @@ class Command(BaseCommand): # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running load_transition_domain script") + system_exit_on_terminate = True + TerminalHelper.prompt_for_execution(system_exit_on_terminate, command_string, "Running load_transition_domain script") return - self.execute_command(command_string) + TerminalHelper.execute_command(command_string) def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): @@ -289,9 +256,10 @@ class Command(BaseCommand): command_string += "--debug " # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script") + system_exit_on_terminate = True + TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running transfer_transition_domains_to_domains script") return - self.execute_command(command_string) + TerminalHelper.execute_command(command_string) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): @@ -300,45 +268,26 @@ class Command(BaseCommand): command_string = "./manage.py send_domain_invitations -s" # Execute the command string if prompts_enabled: - self.prompt_for_execution(command_string, "Running send_domain_invitations script") + system_exit_on_terminate = True + TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running send_domain_invitations script") return - self.execute_command(command_string) + TerminalHelper.execute_command(command_string) def run_migration_scripts(self, - prompts_enabled: bool, - options): + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse): """Runs the following migration scripts (in order): 1 - imports for trans domains 2 - transfer to domain & domain invitation""" - # Get arguments - sep = options.get("sep") - reset_table = options.get("resetTable") - debug_on = options.get("debug") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) - - # Grab filepath information from the arguments - file_location = options.get("loaderDirectory")+"/" - filenames = options.get("loaderFilenames").split(",") - if len(filenames) < 3: - filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" - {TerminalColors.FAIL} - --loaderFilenames expected 3 filenames to follow it, - but only {len(filenames)} were given: - {filenames_as_string} - - PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN - ============= TERMINATING ============= - {TerminalColors.ENDC} - """) - sys.exit() - domain_contacts_filename = filenames[0] - contacts_filename = filenames[1] - domain_statuses_filename = filenames[2] if prompts_enabled: # Allow the user to inspect the filepath @@ -371,9 +320,9 @@ class Command(BaseCommand): PLEASE Re-Run the script with the correct file location and filenames: EXAMPLE: - docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt + docker compose run -T app ./manage.py test_domain_migration --runMigrations --migrationDirectory /app/tmp --migrationFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - """) + """) # noqa return # Proceed executing the migration scripts @@ -398,9 +347,6 @@ class Command(BaseCommand): f"================== SIMULATING LOGINS ==================" f"{TerminalColors.ENDC}") - # command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins" - - # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff # #DEBUG: # TerminalHelper.print_conditional(debug_on, @@ -426,7 +372,7 @@ class Command(BaseCommand): ): """ Does the following; - 1 - run loader scripts + 1 - run migration scripts 2 - simulate logins 3 - send domain invitations (Emails should be sent to the appropriate users note that all moved domains should now be accessible @@ -441,10 +387,39 @@ class Command(BaseCommand): # should execute. Print some indicators to # the terminal so the user knows what is # enabled. + + + # Get arguments debug_on = options.get("debug") - prompts_enabled = debug_on #TODO: add as argument? - run_loaders_enabled = options.get("runLoaders") - simulate_user_login_enabled = options.get("triggerLogins") + prompts_enabled = options.get("prompt") + run_migrations_enabled = options.get("runMigrations") + # simulate_user_login_enabled = options.get("triggerLogins") + sep = options.get("sep") + reset_table = options.get("resetTable") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) + + # Grab filepath information from the arguments + file_location = options.get("migrationDirectory")+"/" + filenames = options.get("migrationFilenames").split(",") + if len(filenames) < 3: + filenames_as_string = "{}".format(", ".join(map(str, filenames))) + logger.info(f""" + {TerminalColors.FAIL} + --migrationFilenames expected 3 filenames to follow it, + but only {len(filenames)} were given: + {filenames_as_string} + + PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN + ============= TERMINATING ============= + {TerminalColors.ENDC} + """) + sys.exit() + domain_contacts_filename = filenames[0] + contacts_filename = filenames[1] + domain_statuses_filename = filenames[2] + TerminalHelper.print_conditional( debug_on, f"""{TerminalColors.OKCYAN} @@ -454,22 +429,23 @@ class Command(BaseCommand): """ ) TerminalHelper.print_conditional( - run_loaders_enabled, + run_migrations_enabled, f"""{TerminalColors.OKCYAN} - ----------RUNNING LOADERS ON---------- + ----------RUNNING MIGRATIONS ON---------- All migration scripts will be run before analyzing the data. {TerminalColors.ENDC} """ ) TerminalHelper.print_conditional( - run_loaders_enabled, + run_migrations_enabled, f"""{TerminalColors.OKCYAN} ----------TRIGGER LOGINS ON---------- Will be simulating user logins {TerminalColors.ENDC} """ ) + # If a user decides to run all migration # scripts, they may or may not wish to # proceed with analysis of the data depending @@ -481,10 +457,18 @@ class Command(BaseCommand): # portion of the script if desired) prompt_continuation_of_analysis = False - # STEP 1 -- RUN LOADERS + # STEP 1 -- RUN MIGRATIONS # Run migration scripts if specified by user - if run_loaders_enabled: - self.run_migration_scripts(prompts_enabled, options) + if run_migrations_enabled: + self.run_migration_scripts(file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse) prompt_continuation_of_analysis = True # STEP 2 -- SIMULATE LOGINS @@ -497,8 +481,8 @@ class Command(BaseCommand): # automatically execute this as the final step # to ensure Domain Information objects get added # to the database.) - - # if run_loaders_enabled: + + # if run_migrations_enabled: # if prompts_enabled: # simulate_user_login_enabled = TerminalHelper.query_yes_no( # f"""{TerminalColors.FAIL} @@ -511,26 +495,28 @@ class Command(BaseCommand): # prompt_continuation_of_analysis = True # STEP 3 -- SEND INVITES - if prompts_enabled: + proceed_with_sending_invites = run_migrations_enabled + if prompts_enabled and run_migrations_enabled: proceed_with_sending_invites = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with sending user invites? + (Y = proceed, N = skip) {TerminalColors.ENDC}""" ) - if not proceed_with_sending_invites: - return - self.run_send_invites_script(debug_on) - prompt_continuation_of_analysis = True + if proceed_with_sending_invites: + self.run_send_invites_script(debug_on, prompts_enabled) + prompt_continuation_of_analysis = True # STEP 4 -- ANALYZE TABLES & GENERATE REPORT # Analyze tables for corrupt data... - if prompt_continuation_of_analysis & prompts_enabled: + if prompt_continuation_of_analysis and prompts_enabled: # ^ (only prompt if we ran steps 1 and/or 2) analyze_tables = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} Proceed with table analysis? + (Y = proceed, N = exit) {TerminalColors.ENDC}""" ) if not analyze_tables: - return + return self.compare_tables(debug_on) diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index 056b8d447..3697c74e1 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -105,8 +105,8 @@ class Command(BaseCommand): logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - Created {total_new_entries} transition domain entries, - Updated {total_updated_domain_entries} transition domain entries + Created {total_new_entries} domain entries, + Updated {total_updated_domain_entries} domain entries Created {total_domain_invitation_entries} domain invitation entries (NOTE: no invitations are SENT in this script) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 99064346d..7772e8350 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -1,4 +1,6 @@ import logging +import os +import sys logger = logging.getLogger(__name__) @@ -59,3 +61,48 @@ class TerminalHelper: # DEBUG: if print_condition: logger.info(print_statement) + + + def execute_command(command_string:str): + """Executes the given command string""" + + logger.info(f"""{TerminalColors.OKCYAN} + ==== EXECUTING... ==== + {TerminalColors.ENDC}""") + os.system(f"{command_string}") + + + def prompt_for_execution(system_exit_on_terminate: bool, command_string: str, prompt_title: str) -> bool: + """Prompts the user to inspect the given terminal command string + and asks if they wish to execute it. If the user responds (y), + execute the command""" + + action_description_for_selecting_no = "skip" + if system_exit_on_terminate: + action_description_for_selecting_no = "exit" + + # Allow the user to inspect the command string + # and ask if they wish to proceed + proceed_execution = TerminalHelper.query_yes_no( + f"""{TerminalColors.OKCYAN} + ===================================================== + {prompt_title} + ===================================================== + *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** + + {command_string} + {TerminalColors.FAIL} + Proceed? (Y = proceed, N = {action_description_for_selecting_no}) + {TerminalColors.ENDC}""" + ) + + # If the user decided to proceed executing the command, + # run the command for loading transition domains. + # Otherwise, exit this subroutine. + if not proceed_execution: + if system_exit_on_terminate: + sys.exit() + return False + + TerminalHelper.execute_command(command_string) + return True \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index c25181c28..069e5231e 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -5,16 +5,4 @@ NEHRP.GOV|JAKING|tech NEHRP.GOV|JBOONE|billing NELSONCOUNTY-VA.GOV|MKELLEY|admin NELSONCOUNTY-VA.GOV|CWILSON|billing -NELSONCOUNTY-VA.GOV|LMCCADE|tech -NELSONVILLENY.GOV|MBOWMAN|tech -NELSONVILLENY.GOV|PMINNERS|billing -NELSONVILLENY.GOV|CWINWARD|admin -NEMI.GOV|DJS1|tech -NEMI.GOV|EREAD|admin -NEMI.GOV|CHOPKINS|billing -NEPA.GOV|BPAUWELS|admin -NEPA.GOV|CRIBEIRO|billing -NEPA.GOV|DKAUFMAN1|tech -NERSC.GOV|TONEILL|admin -NERSC.GOV|JGERSTLE|billing -NERSC.GOV|RSTROMSNESS|tech \ No newline at end of file +NELSONCOUNTY-VA.GOV|LMCCADE|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index 60d526f2d..1f3cc8998 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,6 +1,4 @@ Anomaly.gov|muahaha| TestDomain.gov|ok| NEHRP.GOV|serverHold| -NELSONCOUNTY-VA.GOV|Hold| -NEMI.GOV|clientHold| -NERSC.GOV|ok| \ No newline at end of file +NELSONCOUNTY-VA.GOV|Hold| \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index b9a9d2c38..8dec33aad 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -4,34 +4,161 @@ from registrar.models import ( User, Domain, DomainInvitation, + TransitionDomain, + DomainInformation, UserDomainRole, ) +from registrar.management.commands.master_domain_migrations import Command as master_migration_command + +from registrar.management.commands.utility.terminal_helper import ( + TerminalHelper, +) class TestLogins(TestCase): - """Test the retrieval of invitations.""" + """Test ......""" def setUp(self): - self.domain, _ = Domain.objects.get_or_create(name="igorville.gov") - self.email = "mayor@igorville.gov" - self.invitation, _ = DomainInvitation.objects.get_or_create( - email=self.email, domain=self.domain - ) - self.user, _ = User.objects.get_or_create(email=self.email) + """ """ + # self.user, _ = User.objects.get_or_create(email=self.email) - # clean out the roles each time - UserDomainRole.objects.all().delete() + # # clean out the roles each time + # UserDomainRole.objects.all().delete() + + def tearDown(self): + super().tearDown() + TransitionDomain.objects.all().delete() + Domain.objects.all().delete() + DomainInvitation.objects.all().delete() + DomainInformation.objects.all().delete() - def test_migration_functions(self): - """ Run the master migration script using local test data """ + def compare_tables(self, + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Verifies that the data loaded correctly.""" + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] + for transition_domain in TransitionDomain.objects.all():# DEBUG: + transition_domain_name = transition_domain.domain_name + transition_domain_email = transition_domain.username + + # Check Domain table + matching_domains = Domain.objects.filter(name=transition_domain_name) + # Check Domain Information table + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + # Check Domain Invitation table + matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), + domain__name=transition_domain_name) + + if len(matching_domains) == 0: + missing_domains.append(transition_domain_name) + elif len(matching_domains) > 1: + duplicate_domains.append(transition_domain_name) + if len(matching_domain_informations) == 0: + missing_domain_informations.append(transition_domain_name) + if len(matching_domain_invitations) == 0: + missing_domain_invites.append(transition_domain_name) + + total_missing_domains = len(missing_domains) + total_duplicate_domains = len(duplicate_domains) + total_missing_domain_informations = len(missing_domain_informations) + total_missing_domain_invitations = len(missing_domain_invites) + + total_transition_domains = len(TransitionDomain.objects.all()) + total_domains = len(Domain.objects.all()) + total_domain_informations = len(DomainInformation.objects.all()) + total_domain_invitations = len(DomainInvitation.objects.all()) + + print(f""" + total_missing_domains = {len(missing_domains)} + total_duplicate_domains = {len(duplicate_domains)} + total_missing_domain_informations = {len(missing_domain_informations)} + total_missing_domain_invitations = {len(missing_domain_invites)} + + total_transition_domains = {len(TransitionDomain.objects.all())} + total_domains = {len(Domain.objects.all())} + total_domain_informations = {len(DomainInformation.objects.all())} + total_domain_invitations = {len(DomainInvitation.objects.all())} + """) + + self.assertTrue(total_missing_domains == expected_missing_domains) + self.assertTrue(total_duplicate_domains == expected_duplicate_domains) + self.assertTrue(total_missing_domain_informations == expected_missing_domain_informations) + self.assertTrue(total_missing_domain_invitations == expected_missing_domain_invitations) + + self.assertTrue(total_transition_domains == expected_total_transition_domains) + self.assertTrue(total_domains == expected_total_domains) + self.assertTrue(total_domain_informations == expected_total_domain_informations) + self.assertTrue(total_domain_invitations == expected_total_domain_invitations) + + def test_master_migration_functions(self): + """ Run the full master migration script using local test data. + NOTE: This is more of an integration test and so far does not + follow best practice of limiting the number of assertions per test. + But for now, this will double-check that the script + works as intended. """ - """ (analyze the tables just like the migration script does, but add assert statements) """ - #TODO: finish me! - self.assertTrue(True) + migration_directory = "/app/registrar/tests/data/" + contacts_filename = "test_contacts.txt" + domain_contacts_filename = "test_domain_contacts.txt" + domain_statuses_filename = "test_domain_statuses.txt" + # STEP 1: Run the master migration script using local test data + master_migration_command.run_load_transition_domain_script(master_migration_command(), + migration_directory, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + "|", + False, + False, + False, + 0) + + # run_master_script_command = "./manage.py master_domain_migrations" + # run_master_script_command += " --runMigrations" + # run_master_script_command += " --migrationDirectory /app/registrar/tests/data" + # run_master_script_command += " --migrationFilenames test_contacts.txt,test_domain_contacts.txt,test_domain_statuses.txt" + # TerminalHelper.execute_command(run_master_script_command) + + # STEP 2: (analyze the tables just like the migration script does, but add assert statements) + expected_total_transition_domains = 8 + expected_total_domains = 4 + expected_total_domain_informations = 0 + expected_total_domain_invitations = 7 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + # we expect 8 missing domain invites since the migration does not auto-login new users + expected_missing_domain_informations = 8 + # we expect 1 missing invite from anomaly.gov (an injected error) + expected_missing_domain_invitations = 1 + self.compare_tables(expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + def test_load_transition_domains(): + """ """ def test_user_logins(self): """A new user's first_login callback retrieves their invitations.""" - self.user.first_login() - self.assertTrue(UserDomainRole.objects.get(user=self.user, domain=self.domain)) + # self.user.first_login() + # self.assertTrue(UserDomainRole.objects.get(user=self.user, domain=self.domain)) From 8a168c0cc3273139bf9a965c8f4742652fbe7c03 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:26:42 -0600 Subject: [PATCH 16/49] Update data_migration.md --- docs/operations/data_migration.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 3e22a64f6..00f84a897 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -155,7 +155,18 @@ From this directory, run the following command: ``` NOTE: This will look for all files in /tmp with the .txt extension, but this can -be changed if you are dealing with different extensions. +be changed if you are dealing with different extensions. For instance, a .tar.gz could be expressed +as `--file_extension tar.gz`. + +If you are using a tar.gz file, you will need to perform one additional step to extract it. +Run the following command from the same directory: +```shell +tar -xvf migrationdata/{FILE_NAME}.tar.gz -C migrationdata/ --strip-components=1 +``` + +*FILE_NAME* - Name of the desired file, ex: exportdata + + #### Manual method If the `cat_files_into_getgov.py` script isn't working, follow these steps instead. From 72ae138991305e54fdc1586bf3914ac37a795939 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 27 Oct 2023 11:22:47 -0600 Subject: [PATCH 17/49] unit tests and integration tests Signed-off-by: CocoByte --- .../commands/load_transition_domain.py | 1 - .../commands/master_domain_migrations.py | 146 +++++++++------- .../commands/utility/terminal_helper.py | 22 +-- .../test_transition_domain_migrations.py | 165 ++++++++++++++---- 4 files changed, 229 insertions(+), 105 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index 3dabd68d9..bd811d56d 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -9,7 +9,6 @@ from django.core.management import BaseCommand from registrar.models import TransitionDomain - from registrar.management.commands.utility.terminal_helper import ( TerminalColors, TerminalHelper, diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index a7cd0dcc2..277b49a94 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -15,6 +15,7 @@ from django.test import Client from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand +from django.core.management import call_command from registrar.models import ( Domain, @@ -98,7 +99,7 @@ class Command(BaseCommand): ) parser.add_argument( "--migrationFilenames", - default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt", + default="escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt", help="""The files used for load_transition_domain migration script. Must appear IN ORDER and separated by commas: domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt @@ -225,12 +226,16 @@ class Command(BaseCommand): debug_on: bool, prompts_enabled: bool, debug_max_entries_to_parse: int): + """Runs the load_transition_domain script""" # Create the command string - command_string = "./manage.py load_transition_domain " - command_string += file_location+domain_contacts_filename + " " - command_string += file_location+contacts_filename + " " - command_string += file_location+domain_statuses_filename + " " + command_script = "load_transition_domain" + command_string = ( + f"./manage.py {command_script} " + f"{file_location+domain_contacts_filename} " + f"{file_location+contacts_filename} " + f"{file_location+domain_statuses_filename} " + ) if sep is not None and sep != "|": command_string += f"--sep {sep} " if reset_table: @@ -240,39 +245,59 @@ class Command(BaseCommand): if debug_max_entries_to_parse > 0: command_string += f"--limitParse {debug_max_entries_to_parse} " + # Execute the command string if prompts_enabled: system_exit_on_terminate = True TerminalHelper.prompt_for_execution(system_exit_on_terminate, command_string, "Running load_transition_domain script") - return - TerminalHelper.execute_command(command_string) + + # TODO: make this somehow run inside TerminalHelper prompt + call_command( + command_script, + f"{file_location+domain_contacts_filename}", + f"{file_location+contacts_filename}", + f"{file_location+domain_statuses_filename}", + sep = sep, + resetTable = reset_table, + debug = debug_on, + limitParse = debug_max_entries_to_parse + ) + + def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" # Create the command string - command_string = "./manage.py transfer_transition_domains_to_domains " + command_script = "transfer_transition_domains_to_domains" + command_string = f"./manage.py {command_script}" if debug_on: command_string += "--debug " # Execute the command string if prompts_enabled: system_exit_on_terminate = True TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running transfer_transition_domains_to_domains script") - return - TerminalHelper.execute_command(command_string) - + + # TODO: make this somehow run inside TerminalHelper prompt + call_command( + command_script + ) def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" # Create the command string... - command_string = "./manage.py send_domain_invitations -s" + command_script = "send_domain_invitations" + command_string = f"./manage.py {command_script} -s" # Execute the command string if prompts_enabled: system_exit_on_terminate = True TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running send_domain_invitations script") - return - TerminalHelper.execute_command(command_string) - + + # TODO: make this somehow run inside TerminalHelper prompt + call_command( + command_script, + s=True + ) def run_migration_scripts(self, file_location, @@ -288,7 +313,6 @@ class Command(BaseCommand): 1 - imports for trans domains 2 - transfer to domain & domain invitation""" - if prompts_enabled: # Allow the user to inspect the filepath # data given in the arguments, and prompt @@ -342,12 +366,12 @@ class Command(BaseCommand): """Simulates logins for users (this will add Domain Information objects to our tables)""" - logger.info(f"" - f"{TerminalColors.OKCYAN}" - f"================== SIMULATING LOGINS ==================" - f"{TerminalColors.ENDC}") + # logger.info(f"" + # f"{TerminalColors.OKCYAN}" + # f"================== SIMULATING LOGINS ==================" + # f"{TerminalColors.ENDC}") - # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff + # for invite in DomainInvitation.objects.all(): #TODO: move to unit test # #DEBUG: # TerminalHelper.print_conditional(debug_on, # f"{TerminalColors.OKCYAN}" @@ -393,32 +417,8 @@ class Command(BaseCommand): debug_on = options.get("debug") prompts_enabled = options.get("prompt") run_migrations_enabled = options.get("runMigrations") - # simulate_user_login_enabled = options.get("triggerLogins") - sep = options.get("sep") - reset_table = options.get("resetTable") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) - - # Grab filepath information from the arguments - file_location = options.get("migrationDirectory")+"/" - filenames = options.get("migrationFilenames").split(",") - if len(filenames) < 3: - filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" - {TerminalColors.FAIL} - --migrationFilenames expected 3 filenames to follow it, - but only {len(filenames)} were given: - {filenames_as_string} - - PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN - ============= TERMINATING ============= - {TerminalColors.ENDC} - """) - sys.exit() - domain_contacts_filename = filenames[0] - contacts_filename = filenames[1] - domain_statuses_filename = filenames[2] + simulate_user_login_enabled = False # TODO: delete? Moving to unit test... options.get("triggerLogins") + TerminalHelper.print_conditional( debug_on, @@ -460,6 +460,34 @@ class Command(BaseCommand): # STEP 1 -- RUN MIGRATIONS # Run migration scripts if specified by user if run_migrations_enabled: + # grab arguments for running migrations + sep = options.get("sep") + reset_table = options.get("resetTable") + debug_max_entries_to_parse = int( + options.get("limitParse") + ) + + # Grab filepath information from the arguments + file_location = options.get("migrationDirectory")+"/" + filenames = options.get("migrationFilenames").split(",") + if len(filenames) < 3: + filenames_as_string = "{}".format(", ".join(map(str, filenames))) + logger.info(f""" + {TerminalColors.FAIL} + --migrationFilenames expected 3 filenames to follow it, + but only {len(filenames)} were given: + {filenames_as_string} + + PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN + ============= TERMINATING ============= + {TerminalColors.ENDC} + """) + sys.exit() + domain_contacts_filename = filenames[0] + contacts_filename = filenames[1] + domain_statuses_filename = filenames[2] + + # Run migration scripts self.run_migration_scripts(file_location, domain_contacts_filename, contacts_filename, @@ -482,24 +510,24 @@ class Command(BaseCommand): # to ensure Domain Information objects get added # to the database.) - # if run_migrations_enabled: - # if prompts_enabled: - # simulate_user_login_enabled = TerminalHelper.query_yes_no( - # f"""{TerminalColors.FAIL} - # Proceed with simulating user logins? - # {TerminalColors.ENDC}""" - # ) - # if not simulate_user_login_enabled: - # return - # self.simulate_user_logins(debug_on) - # prompt_continuation_of_analysis = True + if run_migrations_enabled and simulate_user_login_enabled: + if prompts_enabled: + simulate_user_login_enabled = TerminalHelper.query_yes_no( + f"""{TerminalColors.FAIL} + Proceed with simulating user logins? + {TerminalColors.ENDC}""" + ) + if not simulate_user_login_enabled: + return + self.simulate_user_logins(debug_on) + prompt_continuation_of_analysis = True # STEP 3 -- SEND INVITES proceed_with_sending_invites = run_migrations_enabled if prompts_enabled and run_migrations_enabled: proceed_with_sending_invites = TerminalHelper.query_yes_no( f"""{TerminalColors.FAIL} - Proceed with sending user invites? + Proceed with sending user invites for all transition domains? (Y = proceed, N = skip) {TerminalColors.ENDC}""" ) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 7772e8350..e59586816 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -72,10 +72,14 @@ class TerminalHelper: os.system(f"{command_string}") - def prompt_for_execution(system_exit_on_terminate: bool, command_string: str, prompt_title: str) -> bool: - """Prompts the user to inspect the given terminal command string - and asks if they wish to execute it. If the user responds (y), - execute the command""" + def prompt_for_execution(system_exit_on_terminate: bool, + info_to_inspect: str, + prompt_title: str) -> bool: + """Create to reduce code complexity. + Prompts the user to inspect the given string + and asks if they wish to execute it. + Returns true if the user responds (y), + Returns false if the user responds (n)""" action_description_for_selecting_no = "skip" if system_exit_on_terminate: @@ -88,21 +92,19 @@ class TerminalHelper: ===================================================== {prompt_title} ===================================================== - *** IMPORTANT: VERIFY THE FOLLOWING COMMAND LOOKS CORRECT *** + *** IMPORTANT: VERIFY THE FOLLOWING LOOKS CORRECT *** - {command_string} + {info_to_inspect} {TerminalColors.FAIL} Proceed? (Y = proceed, N = {action_description_for_selecting_no}) {TerminalColors.ENDC}""" ) - # If the user decided to proceed executing the command, - # run the command for loading transition domains. - # Otherwise, exit this subroutine. + # If the user decided to proceed return true. + # Otherwise, either return false or exit this subroutine. if not proceed_execution: if system_exit_on_terminate: sys.exit() return False - TerminalHelper.execute_command(command_string) return True \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 8dec33aad..bc48fa818 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -1,3 +1,4 @@ +from unittest.mock import patch from django.test import TestCase from registrar.models import ( @@ -10,21 +11,22 @@ from registrar.models import ( ) from registrar.management.commands.master_domain_migrations import Command as master_migration_command - -from registrar.management.commands.utility.terminal_helper import ( - TerminalHelper, -) +from django.core.management import call_command class TestLogins(TestCase): - """Test ......""" + """ """ def setUp(self): """ """ - # self.user, _ = User.objects.get_or_create(email=self.email) + # self.load_transition_domain_script = "load_transition_domain", + # self.transfer_script = "transfer_transition_domains_to_domains", + # self.master_script = "load_transition_domain", - # # clean out the roles each time - # UserDomainRole.objects.all().delete() + self.test_data_file_location = "/app/registrar/tests/data" + self.test_domain_contact_filename = "test_domain_contacts.txt" + self.test_contact_filename = "test_contacts.txt" + self.test_domain_status_filename = "test_domain_statuses.txt" def tearDown(self): super().tearDown() @@ -32,6 +34,29 @@ class TestLogins(TestCase): Domain.objects.all().delete() DomainInvitation.objects.all().delete() DomainInformation.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def run_load_domains(self): + call_command( + "load_transition_domain", + f"{self.test_data_file_location}/{self.test_domain_contact_filename}", + f"{self.test_data_file_location}/{self.test_contact_filename}", + f"{self.test_data_file_location}/{self.test_domain_status_filename}", + ) + + def run_transfer_domains(self): + call_command("transfer_transition_domains_to_domains") + + def run_master_script(self): + command = call_command( + "master_domain_migrations", + runMigrations=True, + migrationDirectory=f"{self.test_data_file_location}", + migrationFilenames=(f"{self.test_domain_contact_filename}," + f"{self.test_contact_filename}," + f"{self.test_domain_status_filename}"), + ) def compare_tables(self, expected_total_transition_domains, @@ -102,6 +127,7 @@ class TestLogins(TestCase): self.assertTrue(total_domains == expected_total_domains) self.assertTrue(total_domain_informations == expected_total_domain_informations) self.assertTrue(total_domain_invitations == expected_total_domain_invitations) + def test_master_migration_functions(self): """ Run the full master migration script using local test data. @@ -110,28 +136,23 @@ class TestLogins(TestCase): But for now, this will double-check that the script works as intended. """ - migration_directory = "/app/registrar/tests/data/" - contacts_filename = "test_contacts.txt" - domain_contacts_filename = "test_domain_contacts.txt" - domain_statuses_filename = "test_domain_statuses.txt" + self.run_master_script() - # STEP 1: Run the master migration script using local test data - master_migration_command.run_load_transition_domain_script(master_migration_command(), - migration_directory, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - "|", - False, - False, - False, - 0) - - # run_master_script_command = "./manage.py master_domain_migrations" - # run_master_script_command += " --runMigrations" - # run_master_script_command += " --migrationDirectory /app/registrar/tests/data" - # run_master_script_command += " --migrationFilenames test_contacts.txt,test_domain_contacts.txt,test_domain_statuses.txt" - # TerminalHelper.execute_command(run_master_script_command) + # TODO: instead of patching....there has got to be a way of making sure subsequent commands use the django database + # Patch subroutines for migrations + def side_effect(): + self.run_load_domains() + self.run_transfer_domains() + patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_migration_scripts") + mocked_get = patcher.start() + mocked_get.side_effect = side_effect + # Patch subroutines for sending invitations + def side_effect(): + # TODO: what should happen here? + return + patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_send_invites_script") + mocked_get = patcher.start() + mocked_get.side_effect = side_effect # STEP 2: (analyze the tables just like the migration script does, but add assert statements) expected_total_transition_domains = 8 @@ -154,11 +175,85 @@ class TestLogins(TestCase): expected_missing_domain_informations, expected_missing_domain_invitations, ) + + def test_load_transition_domain(self): + + self.run_load_domains() - def test_load_transition_domains(): - """ """ + # STEP 2: (analyze the tables just like the migration script does, but add assert statements) + expected_total_transition_domains = 8 + expected_total_domains = 0 + expected_total_domain_informations = 0 + expected_total_domain_invitations = 0 - def test_user_logins(self): - """A new user's first_login callback retrieves their invitations.""" - # self.user.first_login() - # self.assertTrue(UserDomainRole.objects.get(user=self.user, domain=self.domain)) + expected_missing_domains = 8 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 8 + expected_missing_domain_invitations = 8 + self.compare_tables(expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + def test_transfer_transition_domains_to_domains(self): + + # TODO: setup manually instead of calling other script + self.run_load_domains() + self.run_transfer_domains() + + # Analyze the tables + expected_total_transition_domains = 8 + expected_total_domains = 4 + expected_total_domain_informations = 0 + expected_total_domain_invitations = 7 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 8 + expected_missing_domain_invitations = 1 + self.compare_tables(expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + + + def test_logins(self): + # TODO: setup manually instead of calling other scripts + self.run_load_domains() + self.run_transfer_domains() + + # Simluate Logins + for invite in DomainInvitation.objects.all(): + # get a user with this email address + user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + user.first_login() + + # Analyze the tables + expected_total_transition_domains = 8 + expected_total_domains = 4 + expected_total_domain_informations = 3 + expected_total_domain_invitations = 7 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 1 + expected_missing_domain_invitations = 1 + self.compare_tables(expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) \ No newline at end of file From b236be7f43ad0d1e33d0dd1239dca181d3495b65 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 27 Oct 2023 11:24:10 -0600 Subject: [PATCH 18/49] missed these changes in the last commit --- .../commands/master_domain_migrations.py | 2 +- .../commands/utility/terminal_helper.py | 9 ------ .../test_transition_domain_migrations.py | 30 +++++++++---------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 277b49a94..1b0623b35 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -296,7 +296,7 @@ class Command(BaseCommand): # TODO: make this somehow run inside TerminalHelper prompt call_command( command_script, - s=True + send_emails=True ) def run_migration_scripts(self, diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index e59586816..d56ca47f1 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -61,15 +61,6 @@ class TerminalHelper: # DEBUG: if print_condition: logger.info(print_statement) - - - def execute_command(command_string:str): - """Executes the given command string""" - - logger.info(f"""{TerminalColors.OKCYAN} - ==== EXECUTING... ==== - {TerminalColors.ENDC}""") - os.system(f"{command_string}") def prompt_for_execution(system_exit_on_terminate: bool, diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index bc48fa818..91c8c471e 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -138,21 +138,21 @@ class TestLogins(TestCase): self.run_master_script() - # TODO: instead of patching....there has got to be a way of making sure subsequent commands use the django database - # Patch subroutines for migrations - def side_effect(): - self.run_load_domains() - self.run_transfer_domains() - patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_migration_scripts") - mocked_get = patcher.start() - mocked_get.side_effect = side_effect - # Patch subroutines for sending invitations - def side_effect(): - # TODO: what should happen here? - return - patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_send_invites_script") - mocked_get = patcher.start() - mocked_get.side_effect = side_effect + # # TODO: instead of patching....there has got to be a way of making sure subsequent commands use the django database + # # Patch subroutines for migrations + # def side_effect(): + # self.run_load_domains() + # self.run_transfer_domains() + # patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_migration_scripts") + # mocked_get = patcher.start() + # mocked_get.side_effect = side_effect + # # Patch subroutines for sending invitations + # def side_effect(): + # # TODO: what should happen here? + # return + # patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_send_invites_script") + # mocked_get = patcher.start() + # mocked_get.side_effect = side_effect # STEP 2: (analyze the tables just like the migration script does, but add assert statements) expected_total_transition_domains = 8 From a33efbe77241e7e63d763aa5fee621769455e14c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:07:27 -0600 Subject: [PATCH 19/49] Ran black for linting --- .../commands/load_transition_domain.py | 3 +- .../commands/master_domain_migrations.py | 313 ++++++++++-------- .../commands/utility/terminal_helper.py | 13 +- .../test_transition_domain_migrations.py | 185 ++++++----- 4 files changed, 285 insertions(+), 229 deletions(-) diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index bd811d56d..687750ed8 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -100,8 +100,7 @@ class Command(BaseCommand): return domain_status_dictionary def get_user_emails_dict( - self, - contacts_filename: str, sep + self, contacts_filename: str, sep ) -> defaultdict[str, str]: """Creates mapping of userId -> emails""" user_emails_dictionary = defaultdict(str) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 1b0623b35..a5660b704 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -27,11 +27,12 @@ from registrar.models import ( from registrar.management.commands.utility.terminal_helper import ( TerminalColors, - TerminalHelper + TerminalHelper, ) logger = logging.getLogger(__name__) + class Command(BaseCommand): help = """ """ @@ -48,8 +49,8 @@ class Command(BaseCommand): > --migrationDirectory /app/tmp --migrationFilenames - The files used for load_transition_domain migration script. - Must appear IN ORDER and comma-delimiteds: + The files used for load_transition_domain migration script. + Must appear IN ORDER and comma-delimiteds: EXAMPLE USAGE: > --migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt where... @@ -76,26 +77,30 @@ class Command(BaseCommand): purposes --resetTable - Used by the migration scripts to trigger a prompt for deleting all table entries. + Used by the migration scripts to trigger a prompt for deleting all table entries. Useful for testing purposes, but USE WITH CAUTION - """ + """ # noqa - line length, impacts readability - parser.add_argument("--runMigrations", + parser.add_argument( + "--runMigrations", help="Runs all scripts (in sequence) for transition domain migrations", - action=argparse.BooleanOptionalAction) - + action=argparse.BooleanOptionalAction, + ) + # --triggerLogins # A boolean (default to true), which triggers running # simulations of user logins for each user in domain invitation - parser.add_argument("--triggerLogins", + parser.add_argument( + "--triggerLogins", help="Simulates a user login for each user in domain invitation", - action=argparse.BooleanOptionalAction) - + action=argparse.BooleanOptionalAction, + ) + # The following file arguments have default values for running in the sandbox parser.add_argument( "--migrationDirectory", default="migrationData", - help="The location of the files used for load_transition_domain migration script" + help="The location of the files used for load_transition_domain migration script", ) parser.add_argument( "--migrationFilenames", @@ -107,10 +112,12 @@ class Command(BaseCommand): where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""" + - domain_statuses_filename is the Data file with domain status information""", ) - parser.add_argument("--sep", default="|", help="Delimiter character for the migration files") + parser.add_argument( + "--sep", default="|", help="Delimiter character for the migration files" + ) parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -127,19 +134,19 @@ class Command(BaseCommand): ) def compare_tables(self, debug_on: bool): - """Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. - + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Produces the following report (printed to the terminal): #1 - Print any domains that exist in the transition_domain table but not in their corresponding domain, domain information or domain invitation tables. #2 - Print which table this domain is missing from - #3- Check for duplicate entries in domain or - domain_information tables and print which are + #3- Check for duplicate entries in domain or + domain_information tables and print which are duplicates and in which tables - """ - + """ + logger.info( f"""{TerminalColors.OKCYAN} ============= BEGINNING ANALYSIS =============== @@ -147,13 +154,13 @@ class Command(BaseCommand): """ ) - #TODO: would filteredRelation be faster? + # TODO: would filteredRelation be faster? missing_domains = [] duplicate_domains = [] missing_domain_informations = [] missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all():# DEBUG: + for transition_domain in TransitionDomain.objects.all(): # DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username @@ -165,22 +172,38 @@ class Command(BaseCommand): # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + matching_domain_informations = DomainInformation.objects.filter( + domain__name=transition_domain_name + ) # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), - domain__name=transition_domain_name) + matching_domain_invitations = DomainInvitation.objects.filter( + email=transition_domain_email.lower(), + domain__name=transition_domain_name, + ) if len(matching_domains) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""", + ) missing_domains.append(transition_domain_name) elif len(matching_domains) > 1: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""", + ) duplicate_domains.append(transition_domain_name) if len(matching_domain_informations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""", + ) missing_domain_informations.append(transition_domain_name) if len(matching_domain_invitations) == 0: - TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""") + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""", + ) missing_domain_invites.append(transition_domain_name) total_missing_domains = len(missing_domains) @@ -189,10 +212,16 @@ class Command(BaseCommand): total_missing_domain_invitations = len(missing_domain_invites) missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) - duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains))) - missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations))) - missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites))) - + duplicate_domains_as_string = "{}".format( + ", ".join(map(str, duplicate_domains)) + ) + missing_domain_informations_as_string = "{}".format( + ", ".join(map(str, missing_domain_informations)) + ) + missing_domain_invites_as_string = "{}".format( + ", ".join(map(str, missing_domain_invites)) + ) + logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED ANALYSIS =============== @@ -215,18 +244,19 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ ) - - def run_load_transition_domain_script(self, - file_location: str, - domain_contacts_filename: str, - contacts_filename: str, - domain_statuses_filename: str, - sep: str, - reset_table: bool, - debug_on: bool, - prompts_enabled: bool, - debug_max_entries_to_parse: int): - + + def run_load_transition_domain_script( + self, + file_location: str, + domain_contacts_filename: str, + contacts_filename: str, + domain_statuses_filename: str, + sep: str, + reset_table: bool, + debug_on: bool, + prompts_enabled: bool, + debug_max_entries_to_parse: int, + ): """Runs the load_transition_domain script""" # Create the command string command_script = "load_transition_domain" @@ -245,11 +275,14 @@ class Command(BaseCommand): if debug_max_entries_to_parse > 0: command_string += f"--limitParse {debug_max_entries_to_parse} " - # Execute the command string if prompts_enabled: system_exit_on_terminate = True - TerminalHelper.prompt_for_execution(system_exit_on_terminate, command_string, "Running load_transition_domain script") + TerminalHelper.prompt_for_execution( + system_exit_on_terminate, + command_string, + "Running load_transition_domain script", + ) # TODO: make this somehow run inside TerminalHelper prompt call_command( @@ -257,16 +290,13 @@ class Command(BaseCommand): f"{file_location+domain_contacts_filename}", f"{file_location+contacts_filename}", f"{file_location+domain_statuses_filename}", - sep = sep, - resetTable = reset_table, - debug = debug_on, - limitParse = debug_max_entries_to_parse - ) + sep=sep, + resetTable=reset_table, + debug=debug_on, + limitParse=debug_max_entries_to_parse, + ) - - - - def run_transfer_script(self, debug_on:bool, prompts_enabled: bool): + def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): """Runs the transfer_transition_domains_to_domains script""" # Create the command string command_script = "transfer_transition_domains_to_domains" @@ -276,13 +306,15 @@ class Command(BaseCommand): # Execute the command string if prompts_enabled: system_exit_on_terminate = True - TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running transfer_transition_domains_to_domains script") - - # TODO: make this somehow run inside TerminalHelper prompt - call_command( - command_script + TerminalHelper.prompt_for_execution( + system_exit_on_terminate, + command_string, + "Running transfer_transition_domains_to_domains script", ) + # TODO: make this somehow run inside TerminalHelper prompt + call_command(command_script) + def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): """Runs the send_domain_invitations script""" # Create the command string... @@ -291,28 +323,31 @@ class Command(BaseCommand): # Execute the command string if prompts_enabled: system_exit_on_terminate = True - TerminalHelper.prompt_for_execution(system_exit_on_terminate,command_string, "Running send_domain_invitations script") - - # TODO: make this somehow run inside TerminalHelper prompt - call_command( - command_script, - send_emails=True + TerminalHelper.prompt_for_execution( + system_exit_on_terminate, + command_string, + "Running send_domain_invitations script", ) - def run_migration_scripts(self, - file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse): - """Runs the following migration scripts (in order): - 1 - imports for trans domains - 2 - transfer to domain & domain invitation""" - + # TODO: make this somehow run inside TerminalHelper prompt + call_command(command_script, send_emails=True) + + def run_migration_scripts( + self, + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse, + ): + """Runs the following migration scripts (in order): + 1 - imports for trans domains + 2 - transfer to domain & domain invitation""" + if prompts_enabled: # Allow the user to inspect the filepath # data given in the arguments, and prompt @@ -335,32 +370,35 @@ class Command(BaseCommand): ) # If the user rejected the filepath information - # as incorrect, prompt the user to provide + # as incorrect, prompt the user to provide # correct file inputs in their original command # prompt and exit this subroutine if not files_are_correct: - logger.info(f""" + logger.info( + f""" {TerminalColors.YELLOW} PLEASE Re-Run the script with the correct file location and filenames: EXAMPLE: docker compose run -T app ./manage.py test_domain_migration --runMigrations --migrationDirectory /app/tmp --migrationFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - """) # noqa + """ + ) # noqa return - - # Proceed executing the migration scripts - self.run_load_transition_domain_script(file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse) - self.run_transfer_script(debug_on, prompts_enabled) + # Proceed executing the migration scripts + self.run_load_transition_domain_script( + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse, + ) + self.run_transfer_script(debug_on, prompts_enabled) def simulate_user_logins(self, debug_on): """Simulates logins for users (this will add @@ -370,7 +408,7 @@ class Command(BaseCommand): # f"{TerminalColors.OKCYAN}" # f"================== SIMULATING LOGINS ==================" # f"{TerminalColors.ENDC}") - + # for invite in DomainInvitation.objects.all(): #TODO: move to unit test # #DEBUG: # TerminalHelper.print_conditional(debug_on, @@ -389,7 +427,6 @@ class Command(BaseCommand): # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") # user.delete() - def handle( self, **options, @@ -400,7 +437,7 @@ class Command(BaseCommand): 2 - simulate logins 3 - send domain invitations (Emails should be sent to the appropriate users note that all moved domains should now be accessible - on django admin for an analyst) + on django admin for an analyst) 4 - analyze the data for transition domains and generate a report """ @@ -412,40 +449,40 @@ class Command(BaseCommand): # the terminal so the user knows what is # enabled. - # Get arguments debug_on = options.get("debug") prompts_enabled = options.get("prompt") run_migrations_enabled = options.get("runMigrations") - simulate_user_login_enabled = False # TODO: delete? Moving to unit test... options.get("triggerLogins") - + simulate_user_login_enabled = ( + False # TODO: delete? Moving to unit test... options.get("triggerLogins") + ) TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} + debug_on, + f"""{TerminalColors.OKCYAN} ----------DEBUG MODE ON---------- Detailed print statements activated. {TerminalColors.ENDC} - """ - ) + """, + ) TerminalHelper.print_conditional( - run_migrations_enabled, - f"""{TerminalColors.OKCYAN} + run_migrations_enabled, + f"""{TerminalColors.OKCYAN} ----------RUNNING MIGRATIONS ON---------- All migration scripts will be run before analyzing the data. {TerminalColors.ENDC} - """ - ) + """, + ) TerminalHelper.print_conditional( - run_migrations_enabled, - f"""{TerminalColors.OKCYAN} + run_migrations_enabled, + f"""{TerminalColors.OKCYAN} ----------TRIGGER LOGINS ON---------- Will be simulating user logins {TerminalColors.ENDC} - """ - ) - + """, + ) + # If a user decides to run all migration # scripts, they may or may not wish to # proceed with analysis of the data depending @@ -463,16 +500,15 @@ class Command(BaseCommand): # grab arguments for running migrations sep = options.get("sep") reset_table = options.get("resetTable") - debug_max_entries_to_parse = int( - options.get("limitParse") - ) + debug_max_entries_to_parse = int(options.get("limitParse")) # Grab filepath information from the arguments - file_location = options.get("migrationDirectory")+"/" + file_location = options.get("migrationDirectory") + "/" filenames = options.get("migrationFilenames").split(",") if len(filenames) < 3: filenames_as_string = "{}".format(", ".join(map(str, filenames))) - logger.info(f""" + logger.info( + f""" {TerminalColors.FAIL} --migrationFilenames expected 3 filenames to follow it, but only {len(filenames)} were given: @@ -481,24 +517,27 @@ class Command(BaseCommand): PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN ============= TERMINATING ============= {TerminalColors.ENDC} - """) + """ + ) sys.exit() domain_contacts_filename = filenames[0] contacts_filename = filenames[1] - domain_statuses_filename = filenames[2] + domain_statuses_filename = filenames[2] # Run migration scripts - self.run_migration_scripts(file_location, - domain_contacts_filename, - contacts_filename, - domain_statuses_filename, - sep, - reset_table, - debug_on, - prompts_enabled, - debug_max_entries_to_parse) + self.run_migration_scripts( + file_location, + domain_contacts_filename, + contacts_filename, + domain_statuses_filename, + sep, + reset_table, + debug_on, + prompts_enabled, + debug_max_entries_to_parse, + ) prompt_continuation_of_analysis = True - + # STEP 2 -- SIMULATE LOGINS # Simulate user login for each user in domain # invitation if specified by user OR if running @@ -513,7 +552,7 @@ class Command(BaseCommand): if run_migrations_enabled and simulate_user_login_enabled: if prompts_enabled: simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} + f"""{TerminalColors.FAIL} Proceed with simulating user logins? {TerminalColors.ENDC}""" ) @@ -521,7 +560,7 @@ class Command(BaseCommand): return self.simulate_user_logins(debug_on) prompt_continuation_of_analysis = True - + # STEP 3 -- SEND INVITES proceed_with_sending_invites = run_migrations_enabled if prompts_enabled and run_migrations_enabled: @@ -530,7 +569,7 @@ class Command(BaseCommand): Proceed with sending user invites for all transition domains? (Y = proceed, N = skip) {TerminalColors.ENDC}""" - ) + ) if proceed_with_sending_invites: self.run_send_invites_script(debug_on, prompts_enabled) prompt_continuation_of_analysis = True @@ -546,5 +585,5 @@ class Command(BaseCommand): {TerminalColors.ENDC}""" ) if not analyze_tables: - return + return self.compare_tables(debug_on) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index d56ca47f1..1bc2cbb33 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -61,14 +61,13 @@ class TerminalHelper: # DEBUG: if print_condition: logger.info(print_statement) - - def prompt_for_execution(system_exit_on_terminate: bool, - info_to_inspect: str, - prompt_title: str) -> bool: + def prompt_for_execution( + system_exit_on_terminate: bool, info_to_inspect: str, prompt_title: str + ) -> bool: """Create to reduce code complexity. Prompts the user to inspect the given string - and asks if they wish to execute it. + and asks if they wish to execute it. Returns true if the user responds (y), Returns false if the user responds (n)""" @@ -97,5 +96,5 @@ class TerminalHelper: if system_exit_on_terminate: sys.exit() return False - - return True \ No newline at end of file + + return True diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 91c8c471e..4bd65ea9a 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -10,9 +10,12 @@ from registrar.models import ( UserDomainRole, ) -from registrar.management.commands.master_domain_migrations import Command as master_migration_command +from registrar.management.commands.master_domain_migrations import ( + Command as master_migration_command, +) from django.core.management import call_command + class TestLogins(TestCase): """ """ @@ -27,7 +30,7 @@ class TestLogins(TestCase): self.test_domain_contact_filename = "test_domain_contacts.txt" self.test_contact_filename = "test_contacts.txt" self.test_domain_status_filename = "test_domain_statuses.txt" - + def tearDown(self): super().tearDown() TransitionDomain.objects.all().delete() @@ -43,49 +46,57 @@ class TestLogins(TestCase): f"{self.test_data_file_location}/{self.test_domain_contact_filename}", f"{self.test_data_file_location}/{self.test_contact_filename}", f"{self.test_data_file_location}/{self.test_domain_status_filename}", - ) + ) def run_transfer_domains(self): call_command("transfer_transition_domains_to_domains") def run_master_script(self): command = call_command( - "master_domain_migrations", - runMigrations=True, - migrationDirectory=f"{self.test_data_file_location}", - migrationFilenames=(f"{self.test_domain_contact_filename}," - f"{self.test_contact_filename}," - f"{self.test_domain_status_filename}"), - ) + "master_domain_migrations", + runMigrations=True, + migrationDirectory=f"{self.test_data_file_location}", + migrationFilenames=( + f"{self.test_domain_contact_filename}," + f"{self.test_contact_filename}," + f"{self.test_domain_status_filename}" + ), + ) - def compare_tables(self, - expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations): - """Does a diff between the transition_domain and the following tables: - domain, domain_information and the domain_invitation. + def compare_tables( + self, + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. Verifies that the data loaded correctly.""" missing_domains = [] duplicate_domains = [] missing_domain_informations = [] missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all():# DEBUG: + for transition_domain in TransitionDomain.objects.all(): # DEBUG: transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table - matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + matching_domain_informations = DomainInformation.objects.filter( + domain__name=transition_domain_name + ) # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(), - domain__name=transition_domain_name) + matching_domain_invitations = DomainInvitation.objects.filter( + email=transition_domain_email.lower(), + domain__name=transition_domain_name, + ) if len(matching_domains) == 0: missing_domains.append(transition_domain_name) @@ -106,7 +117,8 @@ class TestLogins(TestCase): total_domain_informations = len(DomainInformation.objects.all()) total_domain_invitations = len(DomainInvitation.objects.all()) - print(f""" + print( + f""" total_missing_domains = {len(missing_domains)} total_duplicate_domains = {len(duplicate_domains)} total_missing_domain_informations = {len(missing_domain_informations)} @@ -116,26 +128,30 @@ class TestLogins(TestCase): total_domains = {len(Domain.objects.all())} total_domain_informations = {len(DomainInformation.objects.all())} total_domain_invitations = {len(DomainInvitation.objects.all())} - """) + """ + ) self.assertTrue(total_missing_domains == expected_missing_domains) self.assertTrue(total_duplicate_domains == expected_duplicate_domains) - self.assertTrue(total_missing_domain_informations == expected_missing_domain_informations) - self.assertTrue(total_missing_domain_invitations == expected_missing_domain_invitations) + self.assertTrue( + total_missing_domain_informations == expected_missing_domain_informations + ) + self.assertTrue( + total_missing_domain_invitations == expected_missing_domain_invitations + ) self.assertTrue(total_transition_domains == expected_total_transition_domains) self.assertTrue(total_domains == expected_total_domains) self.assertTrue(total_domain_informations == expected_total_domain_informations) self.assertTrue(total_domain_invitations == expected_total_domain_invitations) - - + def test_master_migration_functions(self): - """ Run the full master migration script using local test data. - NOTE: This is more of an integration test and so far does not - follow best practice of limiting the number of assertions per test. - But for now, this will double-check that the script - works as intended. """ - + """Run the full master migration script using local test data. + NOTE: This is more of an integration test and so far does not + follow best practice of limiting the number of assertions per test. + But for now, this will double-check that the script + works as intended.""" + self.run_master_script() # # TODO: instead of patching....there has got to be a way of making sure subsequent commands use the django database @@ -162,22 +178,22 @@ class TestLogins(TestCase): expected_missing_domains = 0 expected_duplicate_domains = 0 - # we expect 8 missing domain invites since the migration does not auto-login new users + # we expect 8 missing domain invites since the migration does not auto-login new users expected_missing_domain_informations = 8 # we expect 1 missing invite from anomaly.gov (an injected error) expected_missing_domain_invitations = 1 - self.compare_tables(expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) - + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + def test_load_transition_domain(self): - self.run_load_domains() # STEP 2: (analyze the tables just like the migration script does, but add assert statements) @@ -190,18 +206,18 @@ class TestLogins(TestCase): expected_duplicate_domains = 0 expected_missing_domain_informations = 8 expected_missing_domain_invitations = 8 - self.compare_tables(expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) - + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + def test_transfer_transition_domains_to_domains(self): - # TODO: setup manually instead of calling other script self.run_load_domains() self.run_transfer_domains() @@ -216,28 +232,30 @@ class TestLogins(TestCase): expected_duplicate_domains = 0 expected_missing_domain_informations = 8 expected_missing_domain_invitations = 1 - self.compare_tables(expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) - + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) def test_logins(self): - # TODO: setup manually instead of calling other scripts + # TODO: setup manually instead of calling other scripts self.run_load_domains() self.run_transfer_domains() - + # Simluate Logins for invite in DomainInvitation.objects.all(): # get a user with this email address - user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) + user, user_created = User.objects.get_or_create( + email=invite.email, username=invite.email + ) user.first_login() - + # Analyze the tables expected_total_transition_domains = 8 expected_total_domains = 4 @@ -248,12 +266,13 @@ class TestLogins(TestCase): expected_duplicate_domains = 0 expected_missing_domain_informations = 1 expected_missing_domain_invitations = 1 - self.compare_tables(expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) \ No newline at end of file + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) From 7e04179c2011827866bfb7c742daefbda29ee7fd Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 30 Oct 2023 23:59:50 -0600 Subject: [PATCH 20/49] Linted --- .../commands/master_domain_migrations.py | 125 ++++++------------ .../commands/utility/terminal_helper.py | 1 - .../test_transition_domain_migrations.py | 31 +---- 3 files changed, 45 insertions(+), 112 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index a5660b704..206b63276 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -8,11 +8,6 @@ import logging import argparse import sys -import os - -from django.test import Client - -from django_fsm import TransitionNotAllowed # type: ignore from django.core.management import BaseCommand from django.core.management import call_command @@ -22,7 +17,6 @@ from registrar.models import ( DomainInformation, DomainInvitation, TransitionDomain, - User, ) from registrar.management.commands.utility.terminal_helper import ( @@ -100,19 +94,26 @@ class Command(BaseCommand): parser.add_argument( "--migrationDirectory", default="migrationData", - help="The location of the files used for load_transition_domain migration script", + help=( + "The location of the files used for" + "load_transition_domain migration script" + ), ) parser.add_argument( "--migrationFilenames", - default="escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt", - help="""The files used for load_transition_domain migration script. - Must appear IN ORDER and separated by commas: + default="""escrow_domain_contacts.daily.gov.GOV.txt, + escrow_contacts.daily.gov.GOV.txt, + escrow_domain_statuses.daily.gov.GOV.txt""", + help="""The files used for load_transition_domain migration script. + Must appear IN ORDER and separated by commas: domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt - + where... - - domain_contacts_filename is the Data file with domain contact information + - domain_contacts_filename is the Data file with domain contact + information - contacts_filename is the Data file with contact information - - domain_statuses_filename is the Data file with domain status information""", + - domain_statuses_filename is the Data file with domain status + information""", ) parser.add_argument( @@ -196,13 +197,15 @@ class Command(BaseCommand): if len(matching_domain_informations) == 0: TerminalHelper.print_conditional( debug_on, - f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""", + f"""{TerminalColors.YELLOW}Missing Domain Information + {TerminalColors.ENDC}""", ) missing_domain_informations.append(transition_domain_name) if len(matching_domain_invitations) == 0: TerminalHelper.print_conditional( debug_on, - f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""", + f"""{TerminalColors.YELLOW}Missing Domain Invitation + {TerminalColors.ENDC}""", ) missing_domain_invites.append(transition_domain_name) @@ -225,22 +228,26 @@ class Command(BaseCommand): logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED ANALYSIS =============== - + {total_missing_domains} Missing Domains: (These are transition domains that are missing from the Domain Table) - {TerminalColors.YELLOW}{missing_domains_as_string}{TerminalColors.OKGREEN} - + {TerminalColors.YELLOW}{missing_domains_as_string} + {TerminalColors.OKGREEN} {total_duplicate_domains} Duplicate Domains: - (These are transition domains which have duplicate entries in the Domain Table) - {TerminalColors.YELLOW}{duplicate_domains_as_string}{TerminalColors.OKGREEN} - + (These are transition domains which have duplicate + entries in the Domain Table) + {TerminalColors.YELLOW}{duplicate_domains_as_string} + {TerminalColors.OKGREEN} {total_missing_domain_informations} Domain Information Entries missing: - (These are transition domains which have no entries in the Domain Information Table) - {TerminalColors.YELLOW}{missing_domain_informations_as_string}{TerminalColors.OKGREEN} - + (These are transition domains which have no entries + in the Domain Information Table) + {TerminalColors.YELLOW}{missing_domain_informations_as_string} + {TerminalColors.OKGREEN} {total_missing_domain_invitations} Domain Invitations missing: - (These are transition domains which have no entires in the Domain Invitation Table) - {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} + (These are transition domains which have no entires in + the Domain Invitation Table) + {TerminalColors.YELLOW}{missing_domain_invites_as_string} + {TerminalColors.OKGREEN} {TerminalColors.ENDC} """ ) @@ -377,13 +384,10 @@ class Command(BaseCommand): logger.info( f""" {TerminalColors.YELLOW} - PLEASE Re-Run the script with the correct file location and filenames: - - EXAMPLE: - docker compose run -T app ./manage.py test_domain_migration --runMigrations --migrationDirectory /app/tmp --migrationFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt - + PLEASE Re-Run the script with the correct + file location and filenames: """ - ) # noqa + ) return # Proceed executing the migration scripts @@ -400,33 +404,6 @@ class Command(BaseCommand): ) self.run_transfer_script(debug_on, prompts_enabled) - def simulate_user_logins(self, debug_on): - """Simulates logins for users (this will add - Domain Information objects to our tables)""" - - # logger.info(f"" - # f"{TerminalColors.OKCYAN}" - # f"================== SIMULATING LOGINS ==================" - # f"{TerminalColors.ENDC}") - - # for invite in DomainInvitation.objects.all(): #TODO: move to unit test - # #DEBUG: - # TerminalHelper.print_conditional(debug_on, - # f"{TerminalColors.OKCYAN}" - # f"Processing invite: {invite}" - # f"{TerminalColors.ENDC}") - # # get a user with this email address - # user, user_created = User.objects.get_or_create(email=invite.email, username=invite.email) - # #DEBUG: - # TerminalHelper.print_conditional(user_created, - # f"""{TerminalColors.OKCYAN}No user found (creating temporary user object){TerminalColors.ENDC}""") - # TerminalHelper.print_conditional(debug_on, - # f"""{TerminalColors.OKCYAN}Executing first-time login for user: {user}{TerminalColors.ENDC}""") - # user.first_login() - # if user_created: - # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") - # user.delete() - def handle( self, **options, @@ -453,9 +430,6 @@ class Command(BaseCommand): debug_on = options.get("debug") prompts_enabled = options.get("prompt") run_migrations_enabled = options.get("runMigrations") - simulate_user_login_enabled = ( - False # TODO: delete? Moving to unit test... options.get("triggerLogins") - ) TerminalHelper.print_conditional( debug_on, @@ -538,30 +512,7 @@ class Command(BaseCommand): ) prompt_continuation_of_analysis = True - # STEP 2 -- SIMULATE LOGINS - # Simulate user login for each user in domain - # invitation if specified by user OR if running - # migration scripts. - # (NOTE: Although users can choose to run login - # simulations separately (for testing purposes), - # if we are running all migration scripts, we should - # automatically execute this as the final step - # to ensure Domain Information objects get added - # to the database.) - - if run_migrations_enabled and simulate_user_login_enabled: - if prompts_enabled: - simulate_user_login_enabled = TerminalHelper.query_yes_no( - f"""{TerminalColors.FAIL} - Proceed with simulating user logins? - {TerminalColors.ENDC}""" - ) - if not simulate_user_login_enabled: - return - self.simulate_user_logins(debug_on) - prompt_continuation_of_analysis = True - - # STEP 3 -- SEND INVITES + # STEP 2 -- SEND INVITES proceed_with_sending_invites = run_migrations_enabled if prompts_enabled and run_migrations_enabled: proceed_with_sending_invites = TerminalHelper.query_yes_no( @@ -574,7 +525,7 @@ class Command(BaseCommand): self.run_send_invites_script(debug_on, prompts_enabled) prompt_continuation_of_analysis = True - # STEP 4 -- ANALYZE TABLES & GENERATE REPORT + # STEP 3 -- ANALYZE TABLES & GENERATE REPORT # Analyze tables for corrupt data... if prompt_continuation_of_analysis and prompts_enabled: # ^ (only prompt if we ran steps 1 and/or 2) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 1bc2cbb33..625599b8b 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -1,5 +1,4 @@ import logging -import os import sys logger = logging.getLogger(__name__) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4bd65ea9a..11324cb37 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -1,4 +1,3 @@ -from unittest.mock import patch from django.test import TestCase from registrar.models import ( @@ -10,9 +9,6 @@ from registrar.models import ( UserDomainRole, ) -from registrar.management.commands.master_domain_migrations import ( - Command as master_migration_command, -) from django.core.management import call_command @@ -52,7 +48,7 @@ class TestLogins(TestCase): call_command("transfer_transition_domains_to_domains") def run_master_script(self): - command = call_command( + call_command( "master_domain_migrations", runMigrations=True, migrationDirectory=f"{self.test_data_file_location}", @@ -154,23 +150,8 @@ class TestLogins(TestCase): self.run_master_script() - # # TODO: instead of patching....there has got to be a way of making sure subsequent commands use the django database - # # Patch subroutines for migrations - # def side_effect(): - # self.run_load_domains() - # self.run_transfer_domains() - # patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_migration_scripts") - # mocked_get = patcher.start() - # mocked_get.side_effect = side_effect - # # Patch subroutines for sending invitations - # def side_effect(): - # # TODO: what should happen here? - # return - # patcher = patch("registrar.management.commands.master_domain_migrations.Command.run_send_invites_script") - # mocked_get = patcher.start() - # mocked_get.side_effect = side_effect - - # STEP 2: (analyze the tables just like the migration script does, but add assert statements) + # STEP 2: (analyze the tables just like the + # migration script does, but add assert statements) expected_total_transition_domains = 8 expected_total_domains = 4 expected_total_domain_informations = 0 @@ -178,7 +159,8 @@ class TestLogins(TestCase): expected_missing_domains = 0 expected_duplicate_domains = 0 - # we expect 8 missing domain invites since the migration does not auto-login new users + # we expect 8 missing domain invites since the + # migration does not auto-login new users expected_missing_domain_informations = 8 # we expect 1 missing invite from anomaly.gov (an injected error) expected_missing_domain_invitations = 1 @@ -196,7 +178,8 @@ class TestLogins(TestCase): def test_load_transition_domain(self): self.run_load_domains() - # STEP 2: (analyze the tables just like the migration script does, but add assert statements) + # STEP 2: (analyze the tables just like the migration + # script does, but add assert statements) expected_total_transition_domains = 8 expected_total_domains = 0 expected_total_domain_informations = 0 From 644034fe96328424a230282ac2dc230d7599579b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 31 Oct 2023 10:22:45 -0600 Subject: [PATCH 21/49] linted (except for security flags) --- src/registrar/management/commands/utility/terminal_helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index 625599b8b..7327ef3bd 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -22,6 +22,7 @@ class TerminalColors: class TerminalHelper: + @staticmethod def query_yes_no(question: str, default="yes") -> bool: """Ask a yes/no question via raw_input() and return their answer. @@ -52,6 +53,7 @@ class TerminalHelper: else: logger.info("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + @staticmethod def print_conditional(print_condition: bool, print_statement: str): """This function reduces complexity of debug statements in other functions. @@ -61,12 +63,13 @@ class TerminalHelper: if print_condition: logger.info(print_statement) + @staticmethod def prompt_for_execution( system_exit_on_terminate: bool, info_to_inspect: str, prompt_title: str ) -> bool: """Create to reduce code complexity. Prompts the user to inspect the given string - and asks if they wish to execute it. + and asks if they wish to proceed. Returns true if the user responds (y), Returns false if the user responds (n)""" From 29a422d114d28b671777d39db85bd6dffd9c051b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:40:18 -0600 Subject: [PATCH 22/49] Fixed SEC issue --- .../commands/cat_files_into_getgov.py | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 5de44ba73..ceb7ba6b2 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -3,9 +3,12 @@ import glob import logging import os +import string from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import TerminalHelper + logger = logging.getLogger(__name__) @@ -27,6 +30,7 @@ class Command(BaseCommand): def handle(self, **options): file_extension: str = options.get("file_extension").lstrip(".") directory = options.get("directory") + helper = TerminalHelper() # file_extension is always coerced as str, Truthy is OK to use here. if not file_extension or not isinstance(file_extension, str): @@ -38,29 +42,63 @@ class Command(BaseCommand): for src_file_path in matching_extensions: filename = os.path.basename(src_file_path) + exit_status = -1 do_command = True - exit_status: int desired_file_path = f"{directory}/{filename}" if os.path.exists(desired_file_path): # For linter - prompt = " Do you want to replace it? (y/n) " - replace = input(f"{desired_file_path} already exists. {prompt}") - if replace.lower() != "y": + prompt = "Do you want to replace it?" + replace = f"{desired_file_path} already exists. {prompt}" + if not helper.query_yes_no(replace): do_command = False - if do_command: - copy_from = f"../tmp/{filename}" - self.cat(copy_from, desired_file_path) - exit_status = os.system(f"cat ../tmp/{filename} > {desired_file_path}") - - if exit_status == 0: - logger.info(f"Successfully copied {filename}") - else: - logger.error(f"Failed to copy {filename}") + try: + if do_command: + copy_from = f"../tmp/{filename}" + exit_status = self.cat(copy_from, desired_file_path) + except ValueError as err: + raise err + finally: + if exit_status == 0: + logger.info(f"Successfully copied {filename}") + else: + logger.error(f"Failed to copy {filename}") def cat(self, copy_from, copy_to): """Runs the cat command to copy_from a location to copy_to a location""" - exit_status = os.system(f"cat {copy_from} > {copy_to}") + + # copy_from will be system defined + self.check_file_path(copy_from, check_directory = False) + self.check_file_path(copy_to) + + # This command can only be ran from inside cf ssh getgov-{sandbox} + # It has no utility when running locally, and to exploit this + # you would have to have ssh access anyway, which is a bigger problem. + exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec return exit_status + + def check_file_path(self, file_path: str, check_directory = True): + """Does a check on user input to ensure validity""" + if not isinstance(file_path, str): + raise ValueError("Invalid path provided") + + # Remove any initial/final whitespace + file_path = file_path.strip() + + # Check for any attempts to move up in the directory structure + if ".." in file_path and check_directory: + raise ValueError( + "Moving up in the directory structure is not allowed" + ) + + # Check for any invalid characters + valid_chars = f"/-_.() {string.ascii_letters}{string.digits}" + for char in file_path: + if char not in valid_chars: + raise ValueError( + f"Invalid character {char} in file path" + ) + + return file_path From e1257608db73b9508ec24169c64306a8385732dc Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:47:22 -0600 Subject: [PATCH 23/49] Fix linter issues --- .../management/commands/cat_files_into_getgov.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index ceb7ba6b2..6c46994ea 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -70,35 +70,31 @@ class Command(BaseCommand): copy_from a location to copy_to a location""" # copy_from will be system defined - self.check_file_path(copy_from, check_directory = False) + self.check_file_path(copy_from, check_directory=False) self.check_file_path(copy_to) # This command can only be ran from inside cf ssh getgov-{sandbox} # It has no utility when running locally, and to exploit this # you would have to have ssh access anyway, which is a bigger problem. - exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec + exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec return exit_status - def check_file_path(self, file_path: str, check_directory = True): + def check_file_path(self, file_path: str, check_directory=True): """Does a check on user input to ensure validity""" if not isinstance(file_path, str): raise ValueError("Invalid path provided") - + # Remove any initial/final whitespace file_path = file_path.strip() # Check for any attempts to move up in the directory structure if ".." in file_path and check_directory: - raise ValueError( - "Moving up in the directory structure is not allowed" - ) + raise ValueError("Moving up in the directory structure is not allowed") # Check for any invalid characters valid_chars = f"/-_.() {string.ascii_letters}{string.digits}" for char in file_path: if char not in valid_chars: - raise ValueError( - f"Invalid character {char} in file path" - ) + raise ValueError(f"Invalid character {char} in file path") return file_path From 81cc16d1b3575bd7ad6ca532a144aee5babff423 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 31 Oct 2023 12:08:08 -0600 Subject: [PATCH 24/49] Updated documentation --- docs/operations/data_migration.md | 113 ++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 00f84a897..316098378 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -87,12 +87,12 @@ FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains a We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. -## OPTION 1: SANDBOX -## Load migration data onto a production or sandbox environment +### SANDBOX MIGRATION SETUP +### Load migration data onto a production or sandbox environment **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. Do not use this method to store data you want to keep around permanently. -### STEP 1: Use scp to transfer data +#### STEP 1: Use scp to transfer data CloudFoundry supports scp as means of transferring data locally to our environment. If you are dealing with a batch of files, try sending across a tar.gz and unpacking that. **Login to Cloud.gov** @@ -134,7 +134,7 @@ Copy this code into the password prompt from earlier. NOTE: You can use different utilities to copy this onto the clipboard for you. If you are on Windows, try the command `cf ssh-code | clip`. On Mac, this will be `cf ssh-code | pbcopy` -### STEP 2: Transfer uploaded files to the getgov directory +#### STEP 2: Transfer uploaded files to the getgov directory Due to the nature of how Cloud.gov operates, the getgov directory is dynamically generated whenever the app is built under the tmp/ folder. We can directly upload files to the tmp/ folder but cannot target the generated getgov folder directly, as we need to spin up a shell to access this. From here, we can move those uploaded files into the getgov directory using the `cat` command. Note that you will have to repeat this for each file you want to move, so it is better to use a tar.gz for multiple, and unpack it inside of the `datamigration` folder. **SSH into your sandbox** @@ -168,7 +168,7 @@ tar -xvf migrationdata/{FILE_NAME}.tar.gz -C migrationdata/ --strip-components=1 -#### Manual method +##### Manual method If the `cat_files_into_getgov.py` script isn't working, follow these steps instead. **Move the desired file into the correct directory** @@ -178,34 +178,37 @@ cat ../tmp/{filename} > migrationdata/{filename} ``` -### STEP 3: Load Transition Domain data into TransitionDomain table -Run the following script to transfer the existing data on our .txt files to our DB. -```shell -./manage.py load_transition_domain migrationdata/escrow_domain_contacts.daily.gov.GOV.txt migrationdata/escrow_contacts.daily.gov.GOV.txt migrationdata/escrow_domain_statuses.daily.gov.GOV.txt +#### You are now ready to run migration scripts (see "Running the Migration Scripts") ``` -## OPTION 2: LOCAL -## Load migration data onto our local environments +``` +### LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) +### Load migration data onto our local environments -Transferring this data from these files into our domain tables happens in two steps; +***IMPORTANT: only use test data, to avoid publicizing PII in our public repo.*** -***IMPORTANT: only run the following locally, to avoid publicizing PII in our public repo.*** - -### STEP 1: Load Transition Domain data into TransitionDomain table - -**SETUP** -In order to use the management command, we need to add the files to a folder under `src/`. +In order to run the scripts locally, we need to add the files to a folder under `src/`. This will allow Docker to mount the files to a container (under `/app`) for our use. - Create a folder called `tmp` underneath `src/` - Add the above files to this folder - Open a terminal and navigate to `src/` -Then run the following command (This will parse the three files in your `tmp` folder and load the information into the TransitionDomain table); +#### You are now ready to run migration scripts (see "Running the Migration Scripts") + +### RUNNING THE MIGRATION SCRIPTS + +**NOTE: Whil we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.** + +#### 1 - Load Transition Domains +Run the following command (This will parse the three files in your `tmp` folder and load the information into the TransitionDomain table); ```shell -docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt +docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` +**For Sandbox**: +Change "/app/tmp" to point to the sandbox directory + **OPTIONAL COMMAND LINE ARGUMENTS**: `--debug` This will print out additional, detailed logs. @@ -214,7 +217,7 @@ This will print out additional, detailed logs. Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. `--resetTable` -This will delete all the data loaded into transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. +This will delete all the data in transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. ### STEP 2: Transfer Transition Domain data into main Domain tables @@ -224,7 +227,7 @@ Now that we've loaded all the data into TransitionDomain, we need to update the In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): ```shell -docker compose run -T app ./manage.py transfer_transition_domains_to_domains +docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` **OPTIONAL COMMAND LINE ARGUMENTS**: @@ -232,4 +235,68 @@ docker compose run -T app ./manage.py transfer_transition_domains_to_domains This will print out additional, detailed logs. `--limitParse 100` -Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. \ No newline at end of file +Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. + +### STEP 3: Send Domain invitations +### Run the send invitations script + +To send invitations for every transition domain in the transition domain table, execute the following command: +`docker compose run -T app send_domain_invitations -s` + +### STEP 4: Test the results +### Run the migration analyzer + +This script's main function is to scan the transition domain and domain tables for any anomalies. It produces a simple report of missing or duplicate data. NOTE: some missing data might be expected depending on the nature of our migrations so use best judgement when evaluating the results. + +**ANALYZE ONLY** +To analyze our database without running migrations, execute the script without any optional arguments: +`docker compose run -T app ./manage.py master_domain_migrations --debug` + +**RUN MIGRATIONS FEATURE** +To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: +`docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt` + +#### OPTIONAL ARGUMENTS +`--runMigrations` +A boolean (default to true), which triggers running +all scripts (in sequence) for transition domain migrations + +`--migrationDirectory` +**default="migrationData"** (<--This is the sandbox directory) +The location of the files used for load_transition_domain migration script +EXAMPLE USAGE: +--migrationDirectory /app/tmp + +`--migrationFilenames` +**default=escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt** (<--These are the usual names for the files. The script will throw a warning if it cannot find these exact files, in which case you will need to supply the correct filenames) +The files used for load_transition_domain migration script. +Must appear IN ORDER and comma-delimited: +EXAMPLE USAGE: +--migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt +where... +- domain_contacts_filename is the Data file with domain contact information +- contacts_filename is the Data file with contact information +- domain_statuses_filename is the Data file with domain status information + +`--sep` +Delimiter for the migration scripts to correctly parse the given text files. +(usually this can remain at default value of |) + +`--debug` +A boolean (default to true), which activates additional print statements + +`--prompt` +A boolean (default to true), which activates terminal prompts +that allows the user to step through each portion of this +script. + +`--limitParse` +Used by the migration scripts (load_transition_domain) to set the limit for the +number of data entries to insert. Set to 0 (or just don't use this +argument) to parse every entry. This was provided primarily for testing +purposes + +`--resetTable` +Used by the migration scripts to trigger a prompt for deleting all table entries. +Useful for testing purposes, but USE WITH CAUTION +``` \ No newline at end of file From a69941ffa8413f8ea4a54e00fbfe6dfe88c66121 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 31 Oct 2023 12:15:55 -0600 Subject: [PATCH 25/49] Updating documentation to be more readable --- docs/operations/data_migration.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 316098378..ac58af1a5 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -198,18 +198,20 @@ This will allow Docker to mount the files to a container (under `/app`) for our ### RUNNING THE MIGRATION SCRIPTS -**NOTE: Whil we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.** +**NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.** -#### 1 - Load Transition Domains +#### STEP 1 - Load Transition Domains Run the following command (This will parse the three files in your `tmp` folder and load the information into the TransitionDomain table); ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` **For Sandbox**: + Change "/app/tmp" to point to the sandbox directory **OPTIONAL COMMAND LINE ARGUMENTS**: + `--debug` This will print out additional, detailed logs. @@ -220,7 +222,7 @@ Directs the script to load only the first 100 entries into the table. You can a This will delete all the data in transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. -### STEP 2: Transfer Transition Domain data into main Domain tables +#### STEP 2: Transfer Transition Domain data into main Domain tables Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. @@ -237,23 +239,27 @@ This will print out additional, detailed logs. `--limitParse 100` Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. -### STEP 3: Send Domain invitations -### Run the send invitations script +#### STEP 3: Send Domain invitations +Run the send invitations script To send invitations for every transition domain in the transition domain table, execute the following command: `docker compose run -T app send_domain_invitations -s` -### STEP 4: Test the results -### Run the migration analyzer +#### STEP 4: Test the results +Run the migration analyzer This script's main function is to scan the transition domain and domain tables for any anomalies. It produces a simple report of missing or duplicate data. NOTE: some missing data might be expected depending on the nature of our migrations so use best judgement when evaluating the results. -**ANALYZE ONLY** +OPTION 1 - analyze only + To analyze our database without running migrations, execute the script without any optional arguments: + `docker compose run -T app ./manage.py master_domain_migrations --debug` -**RUN MIGRATIONS FEATURE** +OPTION 2 - run migrations + To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: + `docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt` #### OPTIONAL ARGUMENTS @@ -262,17 +268,24 @@ A boolean (default to true), which triggers running all scripts (in sequence) for transition domain migrations `--migrationDirectory` + **default="migrationData"** (<--This is the sandbox directory) + The location of the files used for load_transition_domain migration script EXAMPLE USAGE: --migrationDirectory /app/tmp `--migrationFilenames` + **default=escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt** (<--These are the usual names for the files. The script will throw a warning if it cannot find these exact files, in which case you will need to supply the correct filenames) + The files used for load_transition_domain migration script. Must appear IN ORDER and comma-delimited: + EXAMPLE USAGE: + --migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt + where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information From 2262d947c5b0401cbcfd72cc800ea0042c535a37 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 31 Oct 2023 12:51:36 -0600 Subject: [PATCH 26/49] Update data_migration.md --- docs/operations/data_migration.md | 99 +++++++++++++++---------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index ac58af1a5..10395cc16 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -87,28 +87,29 @@ FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains a We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. -### SANDBOX MIGRATION SETUP -### Load migration data onto a production or sandbox environment +### SECTION 1 - SANDBOX MIGRATION SETUP +Load migration data onto a production or sandbox environment + **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. Do not use this method to store data you want to keep around permanently. #### STEP 1: Use scp to transfer data CloudFoundry supports scp as means of transferring data locally to our environment. If you are dealing with a batch of files, try sending across a tar.gz and unpacking that. -**Login to Cloud.gov** +##### Login to Cloud.gov ```bash cf login -a api.fr.cloud.gov --sso ``` -**Target your workspace** +##### Target your workspace ```bash cf target -o cisa-dotgov -s {SANDBOX_NAME} ``` *SANDBOX_NAME* - Name of your sandbox, ex: za or ab -**Run the scp command** +##### Run the scp command Use the following command to transfer the desired file: ```shell @@ -123,7 +124,7 @@ These are as follows: NOTE: If you'd wish to change what directory these files are uploaded to, you can change `ssh.fr.cloud.gov:tmp/` to `ssh.fr.cloud.gov:{DIRECTORY_YOU_WANT}/`, but be aware that this makes data migration more tricky than it has to be. -**Get a temp auth code** +##### Get a temp auth code The scp command requires a temporary authentication code. Open a new terminal instance (while keeping the current one open), and enter the following command: @@ -137,19 +138,19 @@ NOTE: You can use different utilities to copy this onto the clipboard for you. I #### STEP 2: Transfer uploaded files to the getgov directory Due to the nature of how Cloud.gov operates, the getgov directory is dynamically generated whenever the app is built under the tmp/ folder. We can directly upload files to the tmp/ folder but cannot target the generated getgov folder directly, as we need to spin up a shell to access this. From here, we can move those uploaded files into the getgov directory using the `cat` command. Note that you will have to repeat this for each file you want to move, so it is better to use a tar.gz for multiple, and unpack it inside of the `datamigration` folder. -**SSH into your sandbox** +##### SSH into your sandbox ```shell cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} ``` -**Open a shell** +##### Open a shell ```shell /tmp/lifecycle/shell ``` -From this directory, run the following command: +##### From this directory, run the following command: ```shell ./manage.py cat_files_into_getgov --file_extension txt ``` @@ -168,22 +169,19 @@ tar -xvf migrationdata/{FILE_NAME}.tar.gz -C migrationdata/ --strip-components=1 -##### Manual method +#### Manual method If the `cat_files_into_getgov.py` script isn't working, follow these steps instead. -**Move the desired file into the correct directory** +##### Move the desired file into the correct directory ```shell cat ../tmp/{filename} > migrationdata/{filename} ``` -#### You are now ready to run migration scripts (see "Running the Migration Scripts") -``` +*You are now ready to run migration scripts (see "Running the Migration Scripts")* -``` -### LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) -### Load migration data onto our local environments +### SECTION 2 - LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) ***IMPORTANT: only use test data, to avoid publicizing PII in our public repo.*** @@ -194,23 +192,21 @@ This will allow Docker to mount the files to a container (under `/app`) for our - Add the above files to this folder - Open a terminal and navigate to `src/` -#### You are now ready to run migration scripts (see "Running the Migration Scripts") -### RUNNING THE MIGRATION SCRIPTS +*You are now ready to run migration scripts (see "Running the Migration Scripts")* -**NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.** +### SECTION 3 - RUNNING THE MIGRATION SCRIPTS -#### STEP 1 - Load Transition Domains -Run the following command (This will parse the three files in your `tmp` folder and load the information into the TransitionDomain table); +*NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.* + +#### STEP 1: Load Transition Domains + +Run the following command, making sure the filepaths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in the sandbox, change "/app/tmp" to point to the sandbox directory) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` -**For Sandbox**: - -Change "/app/tmp" to point to the sandbox directory - -**OPTIONAL COMMAND LINE ARGUMENTS**: +##### COMMAND LINE ARGUMENTS: `--debug` This will print out additional, detailed logs. @@ -225,14 +221,14 @@ This will delete all the data in transtion_domain. It is helpful if you want to #### STEP 2: Transfer Transition Domain data into main Domain tables Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. - In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` -**OPTIONAL COMMAND LINE ARGUMENTS**: +##### COMMAND LINE ARGUMENTS: + `--debug` This will print out additional, detailed logs. @@ -240,76 +236,79 @@ This will print out additional, detailed logs. Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. #### STEP 3: Send Domain invitations -Run the send invitations script To send invitations for every transition domain in the transition domain table, execute the following command: `docker compose run -T app send_domain_invitations -s` #### STEP 4: Test the results -Run the migration analyzer + +Run the migration analyzer. This script's main function is to scan the transition domain and domain tables for any anomalies. It produces a simple report of missing or duplicate data. NOTE: some missing data might be expected depending on the nature of our migrations so use best judgement when evaluating the results. -OPTION 1 - analyze only +##### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: - `docker compose run -T app ./manage.py master_domain_migrations --debug` -OPTION 2 - run migrations +##### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: - `docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt` -#### OPTIONAL ARGUMENTS +##### COMMAND LINE ARGUMENTS + `--runMigrations` + A boolean (default to true), which triggers running all scripts (in sequence) for transition domain migrations `--migrationDirectory` -**default="migrationData"** (<--This is the sandbox directory) +The location of the files used for load_transition_domain migration script. +(default is "migrationData" (This is the sandbox directory)) -The location of the files used for load_transition_domain migration script -EXAMPLE USAGE: ---migrationDirectory /app/tmp +Example Usage: +*--migrationDirectory /app/tmp* -`--migrationFilenames` - -**default=escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt** (<--These are the usual names for the files. The script will throw a warning if it cannot find these exact files, in which case you will need to supply the correct filenames) - -The files used for load_transition_domain migration script. -Must appear IN ORDER and comma-delimited: - -EXAMPLE USAGE: - ---migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt +`--migrationFilenames` +The filenames used for load_transition_domain migration script. +Must appear *in oprder* and comma-delimited: +default is "escrow_domain_contacts.daily.gov.GOV.txt,escrow_contacts.daily.gov.GOV.txt,escrow_domain_statuses.daily.gov.GOV.txt" where... - domain_contacts_filename is the Data file with domain contact information - contacts_filename is the Data file with contact information - domain_statuses_filename is the Data file with domain status information +Example Usage: +*--migrationFilenames domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt* + + `--sep` + Delimiter for the migration scripts to correctly parse the given text files. (usually this can remain at default value of |) `--debug` + A boolean (default to true), which activates additional print statements `--prompt` + A boolean (default to true), which activates terminal prompts that allows the user to step through each portion of this script. `--limitParse` + Used by the migration scripts (load_transition_domain) to set the limit for the number of data entries to insert. Set to 0 (or just don't use this argument) to parse every entry. This was provided primarily for testing purposes `--resetTable` + Used by the migration scripts to trigger a prompt for deleting all table entries. Useful for testing purposes, but USE WITH CAUTION -``` \ No newline at end of file +``` From 7da475cbb3fc6ffdea134f7796a346f8f2900df5 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 31 Oct 2023 12:54:33 -0600 Subject: [PATCH 27/49] Update data_migration.md --- docs/operations/data_migration.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 10395cc16..772233075 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -238,7 +238,9 @@ Directs the script to load only the first 100 entries into the table. You can a #### STEP 3: Send Domain invitations To send invitations for every transition domain in the transition domain table, execute the following command: -`docker compose run -T app send_domain_invitations -s` +```shell +docker compose run -T app send_domain_invitations -s +``` #### STEP 4: Test the results @@ -249,12 +251,16 @@ This script's main function is to scan the transition domain and domain tables f ##### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: -`docker compose run -T app ./manage.py master_domain_migrations --debug` +```shell +docker compose run -T app ./manage.py master_domain_migrations --debug +``` ##### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: -`docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt` +```shell +docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt +``` ##### COMMAND LINE ARGUMENTS From 48c11f417aaf3ca98fc05abd34fe9a3efab25a86 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 31 Oct 2023 12:55:02 -0600 Subject: [PATCH 28/49] Update data_migration.md --- docs/operations/data_migration.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 772233075..d04a20631 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -316,5 +316,4 @@ purposes `--resetTable` Used by the migration scripts to trigger a prompt for deleting all table entries. -Useful for testing purposes, but USE WITH CAUTION -``` +Useful for testing purposes, but *use with caution* From d9ab60ee8cb8de0b04efb5d7a5ae57857ae72dbb Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 31 Oct 2023 13:06:13 -0600 Subject: [PATCH 29/49] Update data_migration.md --- docs/operations/data_migration.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index d04a20631..44e83b164 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -242,9 +242,7 @@ To send invitations for every transition domain in the transition domain table, docker compose run -T app send_domain_invitations -s ``` -#### STEP 4: Test the results - -Run the migration analyzer. +#### STEP 4: Test the results (Run the analyzer script) This script's main function is to scan the transition domain and domain tables for any anomalies. It produces a simple report of missing or duplicate data. NOTE: some missing data might be expected depending on the nature of our migrations so use best judgement when evaluating the results. From 20df8d15afb21ba7d52a1b98a7db927167a911c5 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 31 Oct 2023 13:12:42 -0600 Subject: [PATCH 30/49] Update data_migration.md --- docs/operations/data_migration.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 44e83b164..e2af2b2ff 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -78,11 +78,12 @@ An example script using this technique is in docker compose run app ./manage.py load_domain_invitations /app/escrow_domain_contacts.daily.dotgov.GOV.txt /app/escrow_contacts.daily.dotgov.GOV.txt ``` -## Transition Domains +## Transition Domains (Part 1) - Setup Files for Import We are provided with information about Transition Domains in 3 files: -FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain names to contact ID. Domains in this file will usually have 3 contacts each -FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) -FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses + +- FILE 1: **escrow_domain_contacts.daily.gov.GOV.txt** -> has the map of domain names to contact ID. Domains in this file will usually have 3 contacts each +- FILE 2: **escrow_contacts.daily.gov.GOV.txt** -> has the mapping of contact id to contact email address (which is what we care about for sending domain invitations) +- FILE 3: **escrow_domain_statuses.daily.gov.GOV.txt** -> has the map of domains and their statuses We need to run a few scripts to parse these files into our domain tables. We can do this both locally and in a sandbox. @@ -195,11 +196,11 @@ This will allow Docker to mount the files to a container (under `/app`) for our *You are now ready to run migration scripts (see "Running the Migration Scripts")* -### SECTION 3 - RUNNING THE MIGRATION SCRIPTS +## Transition Domains (Part 2) - Running the Migration Scripts *NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.* -#### STEP 1: Load Transition Domains +### STEP 1: Load Transition Domains Run the following command, making sure the filepaths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in the sandbox, change "/app/tmp" to point to the sandbox directory) ```shell @@ -218,7 +219,7 @@ Directs the script to load only the first 100 entries into the table. You can a This will delete all the data in transtion_domain. It is helpful if you want to see the entries reload from scratch or for clearing test data. -#### STEP 2: Transfer Transition Domain data into main Domain tables +### STEP 2: Transfer Transition Domain data into main Domain tables Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. In the same terminal as used in STEP 1, run the command below; @@ -235,25 +236,25 @@ This will print out additional, detailed logs. `--limitParse 100` Directs the script to load only the first 100 entries into the table. You can adjust this number as needed for testing purposes. -#### STEP 3: Send Domain invitations +### STEP 3: Send Domain invitations To send invitations for every transition domain in the transition domain table, execute the following command: ```shell docker compose run -T app send_domain_invitations -s ``` -#### STEP 4: Test the results (Run the analyzer script) +### STEP 4: Test the results (Run the analyzer script) This script's main function is to scan the transition domain and domain tables for any anomalies. It produces a simple report of missing or duplicate data. NOTE: some missing data might be expected depending on the nature of our migrations so use best judgement when evaluating the results. -##### OPTION 1 - ANALYZE ONLY +#### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` -##### OPTION 2 - RUN MIGRATIONS FEATURE +#### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: ```shell From eb9c6fa7d228c98c7d842c31d6077f97261b5a41 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 11:07:12 -0600 Subject: [PATCH 31/49] Update documentation --- docs/operations/data_migration.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index e2af2b2ff..2946228fb 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -94,9 +94,24 @@ Load migration data onto a production or sandbox environment **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. Do not use this method to store data you want to keep around permanently. -#### STEP 1: Use scp to transfer data +#### STEP 1: Using cat to transfer data to sandboxes + +```bash +cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +``` + +* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt +* DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt + + +#### STEP 1 (Alternative): Using scp to transfer data to sandboxes +**IMPORTANT:** Only follow these steps if cat does not work as expected. If it does, skip to step 2. + CloudFoundry supports scp as means of transferring data locally to our environment. If you are dealing with a batch of files, try sending across a tar.gz and unpacking that. + + ##### Login to Cloud.gov ```bash From 8b49117cb195f7df43552fb7beddcc8fcbdee539 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 11:26:16 -0600 Subject: [PATCH 32/49] Fix typo in default migrationDirectory --- src/registrar/management/commands/master_domain_migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 206b63276..bfd34c944 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -93,7 +93,7 @@ class Command(BaseCommand): # The following file arguments have default values for running in the sandbox parser.add_argument( "--migrationDirectory", - default="migrationData", + default="migrationdata", help=( "The location of the files used for" "load_transition_domain migration script" From 6448aae6bbd4a400207c2f7a7f1809dc5ec5f117 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 11:42:23 -0600 Subject: [PATCH 33/49] Hotfix --- .../management/commands/master_domain_migrations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index bfd34c944..3f4841129 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -101,9 +101,9 @@ class Command(BaseCommand): ) parser.add_argument( "--migrationFilenames", - default="""escrow_domain_contacts.daily.gov.GOV.txt, - escrow_contacts.daily.gov.GOV.txt, - escrow_domain_statuses.daily.gov.GOV.txt""", + default="escrow_domain_contacts.daily.gov.GOV.txt," + "escrow_contacts.daily.gov.GOV.txt," + "escrow_domain_statuses.daily.gov.GOV.txt", help="""The files used for load_transition_domain migration script. Must appear IN ORDER and separated by commas: domain_contacts_filename.txt,contacts_filename.txt,domain_statuses_filename.txt From da43be695defd37aab1b2dbf99f3090058a040ff Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:00:27 -0600 Subject: [PATCH 34/49] update docs --- docs/operations/data_migration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 2946228fb..32d349278 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -104,6 +104,7 @@ cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /ho * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt +**TROUBLESHOOTING:** Depending on your operating system (Windows for instance), this command may upload corrupt data. If you encounter the error `gzip: prfiles.tar.gz: not in gzip format` when trying to unzip a .tar.gz file, use the scp command instead. #### STEP 1 (Alternative): Using scp to transfer data to sandboxes **IMPORTANT:** Only follow these steps if cat does not work as expected. If it does, skip to step 2. From 6031e2c3c9c6db95282eb390b6637185d7f7af67 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:39:33 -0600 Subject: [PATCH 35/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 32d349278..425065207 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -92,7 +92,7 @@ We can do this both locally and in a sandbox. Load migration data onto a production or sandbox environment **WARNING:** All files uploaded in this manner are temporary, i.e. they will be deleted when the app is restaged. -Do not use this method to store data you want to keep around permanently. +Do not use these environments to store data you want to keep around permanently. We don't want sensitive data to be accidentally present in our application environments. #### STEP 1: Using cat to transfer data to sandboxes From 27465169aa05eb7678e5e2f0f4e974c08f9089b3 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:41:33 -0600 Subject: [PATCH 36/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 425065207..b577f062d 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -135,7 +135,7 @@ scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HE ``` The items in curly braces are the values that you will manually replace. These are as follows: -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of the app running in your environment, e.g. getgov-za or getgov-stable * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt From a0b405f713a9361e7b3fcca083e174204d277e73 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 12:49:25 -0500 Subject: [PATCH 37/49] Scrubbed test files of any possible PII --- src/registrar/tests/data/test_contacts.txt | 14 +++++++------- .../tests/data/test_domain_contacts.txt | 16 ++++++++-------- .../tests/data/test_domain_statuses.txt | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt index dec8f6816..89f57ccf8 100644 --- a/src/registrar/tests/data/test_contacts.txt +++ b/src/registrar/tests/data/test_contacts.txt @@ -1,7 +1,7 @@ -TESTUSER|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||testuser@gmail.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| -RJD1|52545_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||agustina.wyman7@test.com|GSA|VERISIGN|ctldbatch|2021-06-29T18:53:09Z|VERISIGN|ctldbatch|2021-06-29T18:58:08Z| -JAKING|52555_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||susy.martin4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T15:38:10Z| -JBOONE|52556_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||stephania.winters4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T18:28:09Z| -MKELLEY|52557_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||alexandra.bobbitt5@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-08-02T22:13:09Z| -CWILSON|52562_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||jospeh.mcdowell3@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:33:09Z| -LMCCADE|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||reginald.ratcliff4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file +TESTUSER|12363_CONTACT|123-123-1234||918-000-0000||testuser@gmail.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER1|12345_CONTACT|123-123-1234||918-000-0000||agustina.wyman7@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-29T18:53:09Z|SOMECOMPANY|ctldbatch|2021-06-29T18:58:08Z| +USER2|12355_CONTACT|123-123-1234||918-000-0000||susy.martin4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T15:38:10Z| +USER3|12356_CONTACT|123-123-1234||918-000-0000||stephania.winters4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T18:28:09Z| +USER4|12357_CONTACT|123-123-1234||918-000-0000||alexandra.bobbitt5@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-08-02T22:13:09Z| +USER5|12362_CONTACT|123-123-1234||918-000-0000||jospeh.mcdowell3@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:33:09Z| +USER6|12363_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index 069e5231e..3a1ed745f 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -1,8 +1,8 @@ -Anomaly.gov|ANOMALY|tech -TestDomain.gov|TESTUSER|admin -NEHRP.GOV|RJD1|admin -NEHRP.GOV|JAKING|tech -NEHRP.GOV|JBOONE|billing -NELSONCOUNTY-VA.GOV|MKELLEY|admin -NELSONCOUNTY-VA.GOV|CWILSON|billing -NELSONCOUNTY-VA.GOV|LMCCADE|tech \ No newline at end of file +Anomaly.gov|ANOMALY|tech +TestDomain.gov|TESTUSER|admin +FakeWebsite1|USER1|admin +FakeWebsite1|USER2|tech +FakeWebsite1|USER3|billing +FakeWebsite2.GOV|USER4|admin +FakeWebsite2.GOV|USER5|billing +FakeWebsite2.GOV|USER6|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index 1f3cc8998..021e52ae7 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,4 +1,4 @@ Anomaly.gov|muahaha| -TestDomain.gov|ok| -NEHRP.GOV|serverHold| -NELSONCOUNTY-VA.GOV|Hold| \ No newline at end of file +TestDomain.gov|ok| +FakeWebsite1.GOV|serverHold| +FakeWebsite2.GOV|Hold| \ No newline at end of file From 183a4bf80156c13735d47eebc18efcbd6dcd413b Mon Sep 17 00:00:00 2001 From: CuriousX Date: Thu, 2 Nov 2023 12:58:50 -0500 Subject: [PATCH 38/49] Update data_migration.md --- docs/operations/data_migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..68a6419d5 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of your sandbox, ex: getgov-za * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt From 01d50fe1b32b166627ee96d4ffb56416b1991aa2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:15:51 -0600 Subject: [PATCH 39/49] Documentation updates / fix script --- docs/operations/data_migration.md | 12 ++-- .../commands/cat_files_into_getgov.py | 64 ++++--------------- 2 files changed, 16 insertions(+), 60 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..1739b5cab 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of the app running in your environment, e.g. getgov-za or getgov-stable * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt @@ -130,7 +130,7 @@ cf target -o cisa-dotgov -s {SANDBOX_NAME} Use the following command to transfer the desired file: ```shell -scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {FULL_NAME_OF_YOUR_SANDBOX_HERE} --guid)/processes | jq -r '.resources[] +scp -P 2222 -o User=cf:$(cf curl /v3/apps/$(cf app {APP_NAME_IN_ENVIRONMENT} --guid)/processes | jq -r '.resources[] | select(.type=="web") | .guid')/0 {LOCAL_PATH_TO_FILE} ssh.fr.cloud.gov:tmp/{DESIRED_NAME_OF_FILE} ``` The items in curly braces are the values that you will manually replace. @@ -139,8 +139,6 @@ These are as follows: * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt -NOTE: If you'd wish to change what directory these files are uploaded to, you can change `ssh.fr.cloud.gov:tmp/` to `ssh.fr.cloud.gov:{DIRECTORY_YOU_WANT}/`, but be aware that this makes data migration more tricky than it has to be. - ##### Get a temp auth code The scp command requires a temporary authentication code. Open a new terminal instance (while keeping the current one open), @@ -158,7 +156,7 @@ Due to the nature of how Cloud.gov operates, the getgov directory is dynamically ##### SSH into your sandbox ```shell -cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} +cf ssh {APP_NAME_IN_ENVIRONMENT} ``` ##### Open a shell @@ -196,7 +194,7 @@ cat ../tmp/{filename} > migrationdata/{filename} ``` -*You are now ready to run migration scripts (see "Running the Migration Scripts")* +*You are now ready to run migration scripts (see [Running the Migration Scripts](running-the-migration-scripts))* ### SECTION 2 - LOCAL MIGRATION SETUP (TESTING PURPOSES ONLY) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 6c46994ea..c35d2f21a 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -3,6 +3,7 @@ import glob import logging import os +import shutil import string from django.core.management import BaseCommand @@ -36,65 +37,22 @@ class Command(BaseCommand): if not file_extension or not isinstance(file_extension, str): raise ValueError(f"Invalid file extension '{file_extension}'") - matching_extensions = glob.glob(f"../tmp/*.{file_extension}") - if not matching_extensions: + matching_files = glob.glob(f"../tmp/*.{file_extension}") + if not matching_files: logger.error(f"No files with the extension {file_extension} found") + return None - for src_file_path in matching_extensions: + for src_file_path in matching_files: filename = os.path.basename(src_file_path) - exit_status = -1 - do_command = True - - desired_file_path = f"{directory}/{filename}" + + desired_file_path = os.path.join(directory, filename) if os.path.exists(desired_file_path): # For linter prompt = "Do you want to replace it?" replace = f"{desired_file_path} already exists. {prompt}" if not helper.query_yes_no(replace): - do_command = False + continue + + src_file_path = f"../tmp/{filename}" + shutil.copy(src_file_path, desired_file_path) - try: - if do_command: - copy_from = f"../tmp/{filename}" - exit_status = self.cat(copy_from, desired_file_path) - except ValueError as err: - raise err - finally: - if exit_status == 0: - logger.info(f"Successfully copied {filename}") - else: - logger.error(f"Failed to copy {filename}") - - def cat(self, copy_from, copy_to): - """Runs the cat command to - copy_from a location to copy_to a location""" - - # copy_from will be system defined - self.check_file_path(copy_from, check_directory=False) - self.check_file_path(copy_to) - - # This command can only be ran from inside cf ssh getgov-{sandbox} - # It has no utility when running locally, and to exploit this - # you would have to have ssh access anyway, which is a bigger problem. - exit_status = os.system(f"cat {copy_from} > {copy_to}") # nosec - return exit_status - - def check_file_path(self, file_path: str, check_directory=True): - """Does a check on user input to ensure validity""" - if not isinstance(file_path, str): - raise ValueError("Invalid path provided") - - # Remove any initial/final whitespace - file_path = file_path.strip() - - # Check for any attempts to move up in the directory structure - if ".." in file_path and check_directory: - raise ValueError("Moving up in the directory structure is not allowed") - - # Check for any invalid characters - valid_chars = f"/-_.() {string.ascii_letters}{string.digits}" - for char in file_path: - if char not in valid_chars: - raise ValueError(f"Invalid character {char} in file path") - - return file_path From 32d5cfac6fb83e07e679a9ed533418d83ba8a6de Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:17:46 -0600 Subject: [PATCH 40/49] Run black --- src/registrar/management/commands/cat_files_into_getgov.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index c35d2f21a..4cd17f6e2 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -44,7 +44,7 @@ class Command(BaseCommand): for src_file_path in matching_files: filename = os.path.basename(src_file_path) - + desired_file_path = os.path.join(directory, filename) if os.path.exists(desired_file_path): # For linter @@ -52,7 +52,6 @@ class Command(BaseCommand): replace = f"{desired_file_path} already exists. {prompt}" if not helper.query_yes_no(replace): continue - + src_file_path = f"../tmp/{filename}" shutil.copy(src_file_path, desired_file_path) - From 4d5f104683de4d87c206537f3ca4f9a95dccc63c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:28:10 -0600 Subject: [PATCH 41/49] Fix md --- docs/operations/data_migration.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 1739b5cab..6a42962e8 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -122,9 +122,9 @@ cf login -a api.fr.cloud.gov --sso ##### Target your workspace ```bash -cf target -o cisa-dotgov -s {SANDBOX_NAME} +cf target -o cisa-dotgov -s {ENVIRONMENT_NAME} ``` -*SANDBOX_NAME* - Name of your sandbox, ex: za or ab +*ENVIRONMENT_NAME* - Name of your sandbox, ex: za or ab ##### Run the scp command @@ -203,8 +203,7 @@ cat ../tmp/{filename} > migrationdata/{filename} In order to run the scripts locally, we need to add the files to a folder under `src/`. This will allow Docker to mount the files to a container (under `/app`) for our use. - - Create a folder called `tmp` underneath `src/` - - Add the above files to this folder + - Add the above files to the `migrationdata/` folder - Open a terminal and navigate to `src/` From 4b7fd9c34b716d33f07e180713715073e17bceb1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:30:12 -0600 Subject: [PATCH 42/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 6a42962e8..3821b0029 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -207,7 +207,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our - Open a terminal and navigate to `src/` -*You are now ready to run migration scripts (see "Running the Migration Scripts")* +*You are now ready to run migration scripts.* ## Transition Domains (Part 2) - Running the Migration Scripts From 997f0c88386da0c8ca4fb31f5903c67172d8b228 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:30:35 -0600 Subject: [PATCH 43/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 3821b0029..2dd00a39f 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -215,7 +215,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our ### STEP 1: Load Transition Domains -Run the following command, making sure the filepaths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in the sandbox, change "/app/tmp" to point to the sandbox directory) +Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` From dcbe41f10791a4fa16cee00db11f647a31108bda Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:33:14 -0600 Subject: [PATCH 44/49] Fix linter issue --- src/registrar/management/commands/cat_files_into_getgov.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/cat_files_into_getgov.py b/src/registrar/management/commands/cat_files_into_getgov.py index 4cd17f6e2..993643ee4 100644 --- a/src/registrar/management/commands/cat_files_into_getgov.py +++ b/src/registrar/management/commands/cat_files_into_getgov.py @@ -4,7 +4,6 @@ import logging import os import shutil -import string from django.core.management import BaseCommand From ea760c76bd9c2e0455f86033d005b5eaea7f2bc0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Nov 2023 13:42:35 -0600 Subject: [PATCH 45/49] Updated documentation for clarity on command arguments and usage of "./manage.py" --- docs/operations/data_migration.md | 26 ++++++++++++------- .../commands/master_domain_migrations.py | 10 +++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 2dd00a39f..3f3302e51 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -215,7 +215,9 @@ This will allow Docker to mount the files to a container (under `/app`) for our ### STEP 1: Load Transition Domains -Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. (NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory) +Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. + +(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and remove "./manage.py" from the command) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` @@ -237,6 +239,8 @@ This will delete all the data in transtion_domain. It is helpful if you want to Now that we've loaded all the data into TransitionDomain, we need to update the main Domain and DomainInvitation tables with this information. In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): + +(NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` @@ -252,8 +256,10 @@ Directs the script to load only the first 100 entries into the table. You can a ### STEP 3: Send Domain invitations To send invitations for every transition domain in the transition domain table, execute the following command: + +(NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell -docker compose run -T app send_domain_invitations -s +docker compose run -T app ./manage.py send_domain_invitations -s ``` ### STEP 4: Test the results (Run the analyzer script) @@ -263,6 +269,8 @@ This script's main function is to scan the transition domain and domain tables f #### OPTION 1 - ANALYZE ONLY To analyze our database without running migrations, execute the script without any optional arguments: + +(NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` @@ -270,6 +278,9 @@ docker compose run -T app ./manage.py master_domain_migrations --debug #### OPTION 2 - RUN MIGRATIONS FEATURE To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: + + +(NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt ``` @@ -278,13 +289,12 @@ docker compose run -T app ./manage.py master_domain_migrations --runMigrations - `--runMigrations` -A boolean (default to true), which triggers running -all scripts (in sequence) for transition domain migrations +Runs all scripts (in sequence) for transition domain migrations `--migrationDirectory` The location of the files used for load_transition_domain migration script. -(default is "migrationData" (This is the sandbox directory)) +(default is "migrationdata" (This is the sandbox directory)) Example Usage: *--migrationDirectory /app/tmp* @@ -310,13 +320,11 @@ Delimiter for the migration scripts to correctly parse the given text files. `--debug` -A boolean (default to true), which activates additional print statements +Activates additional print statements `--prompt` -A boolean (default to true), which activates terminal prompts -that allows the user to step through each portion of this -script. +Activates terminal prompts that allows the user to step through each portion of this script. `--limitParse` diff --git a/src/registrar/management/commands/master_domain_migrations.py b/src/registrar/management/commands/master_domain_migrations.py index 3f4841129..12149ed7b 100644 --- a/src/registrar/management/commands/master_domain_migrations.py +++ b/src/registrar/management/commands/master_domain_migrations.py @@ -34,8 +34,8 @@ class Command(BaseCommand): """ OPTIONAL ARGUMENTS: --runMigrations - A boolean (default to true), which triggers running - all scripts (in sequence) for transition domain migrations + Triggers running all scripts (in sequence) + for transition domain migrations --migrationDirectory The location of the files used for load_transition_domain migration script @@ -57,11 +57,11 @@ class Command(BaseCommand): (usually this can remain at default value of |) --debug - A boolean (default to true), which activates additional print statements + Activates additional print statements --prompt - A boolean (default to true), which activates terminal prompts - that allows the user to step through each portion of this + Activates terminal prompts that allows + the user to step through each portion of this script. --limitParse From 9f03c3ced0c5c98c45eea58cd509d4d330beee6c Mon Sep 17 00:00:00 2001 From: CuriousX Date: Mon, 6 Nov 2023 13:45:31 -0600 Subject: [PATCH 46/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 3f3302e51..6ed24f3ac 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -255,7 +255,7 @@ Directs the script to load only the first 100 entries into the table. You can a ### STEP 3: Send Domain invitations -To send invitations for every transition domain in the transition domain table, execute the following command: +To send invitation emails for every transition domain in the transition domain table, execute the following command: (NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell From 721fc0552093269f7ab5740c2f891e459730619f Mon Sep 17 00:00:00 2001 From: CuriousX Date: Mon, 6 Nov 2023 14:17:16 -0600 Subject: [PATCH 47/49] Update docs/operations/data_migration.md Co-authored-by: Neil MartinsenBurrell --- docs/operations/data_migration.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 6ed24f3ac..c4aad7da4 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -211,7 +211,6 @@ This will allow Docker to mount the files to a container (under `/app`) for our ## Transition Domains (Part 2) - Running the Migration Scripts -*NOTE: While we recommend executing the following scripts individually (Steps 1-3), migrations can also be done 'all at once' using the "Run Migration Feature" in step 4. Use with discretion.* ### STEP 1: Load Transition Domains From 386300fcaa80a0bd16baca3cd379945838a5df8b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Nov 2023 14:52:30 -0600 Subject: [PATCH 48/49] Small change to (hopefully) trigger OWASP to pass --- docs/operations/data_migration.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index c4aad7da4..115a5a68a 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -278,7 +278,6 @@ docker compose run -T app ./manage.py master_domain_migrations --debug To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: - (NOTE: If working in cloud.gov, remove "./manage.py" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt From 350af91ff68c32490c2c4428e24bb5e6b317e8d0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Nov 2023 14:56:36 -0600 Subject: [PATCH 49/49] Updated command instructions in migration doc --- docs/operations/data_migration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 115a5a68a..75bd50a9b 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -216,7 +216,7 @@ This will allow Docker to mount the files to a container (under `/app`) for our Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. -(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and remove "./manage.py" from the command) +(NOTE: If working in cloud.gov, change "/app/tmp" to point to the `migrationdata/` directory and and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py load_transition_domain /app/tmp/escrow_domain_contacts.daily.gov.GOV.txt /app/tmp/escrow_contacts.daily.gov.GOV.txt /app/tmp/escrow_domain_statuses.daily.gov.GOV.txt --debug ``` @@ -239,7 +239,7 @@ Now that we've loaded all the data into TransitionDomain, we need to update the In the same terminal as used in STEP 1, run the command below; (This will parse the data in TransitionDomain and either create a corresponding Domain object, OR, if a corresponding Domain already exists, it will update that Domain with the incoming status. It will also create DomainInvitation objects for each user associated with the domain): -(NOTE: If working in cloud.gov, remove "./manage.py" from the command) +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py transfer_transition_domains_to_domains --debug ``` @@ -256,7 +256,7 @@ Directs the script to load only the first 100 entries into the table. You can a To send invitation emails for every transition domain in the transition domain table, execute the following command: -(NOTE: If working in cloud.gov, remove "./manage.py" from the command) +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py send_domain_invitations -s ``` @@ -269,7 +269,7 @@ This script's main function is to scan the transition domain and domain tables f To analyze our database without running migrations, execute the script without any optional arguments: -(NOTE: If working in cloud.gov, remove "./manage.py" from the command) +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --debug ``` @@ -278,7 +278,7 @@ docker compose run -T app ./manage.py master_domain_migrations --debug To run the migrations again (all above migration steps) before analyzing, execute the following command (read the documentation on the terminal arguments below. Everything used by the migration scripts can also be passed into this script and will have the same effects). NOTE: --debug and --prompt allow you to step through the migration process and exit it after each step if you need to. It is recommended that you use these arguments when using the --runMigrations feature: -(NOTE: If working in cloud.gov, remove "./manage.py" from the command) +(NOTE: If working in cloud.gov, and remove "docker compose run -T app" from the command) ```shell docker compose run -T app ./manage.py master_domain_migrations --runMigrations --debug --prompt ```