zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-07-11 21:48:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

file renames

Browse source 
Signed-off-by: CocoByte <nicolle.leclair@gmail.com>
This commit is contained in:
CocoByte 2023-10-25 11:26:26 -06:00
parent 53c74a766f
commit b62a9b4223
No known key found for this signature in database
GPG key ID: BBFAA2526384C97F
2 changed files with 146 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

207
src/registrar/management/commands/full_domain_migrations.py → src/registrar/management/commands/master_domain_migrations.py
View file

@ -10,6 +10,10 @@ import argparse
import sys import sys
import os import os
from django.test import Client
from django_fsm import TransitionNotAllowed # type: ignore
from django.core.management import BaseCommand from django.core.management import BaseCommand
from registrar.models import ( from registrar.models import (
@ -17,26 +21,19 @@ from registrar.models import (
DomainInformation, DomainInformation,
DomainInvitation, DomainInvitation,
TransitionDomain, TransitionDomain,
User,
) )
from registrar.management.commands.utility.terminal_helper import ( from registrar.management.commands.utility.terminal_helper import (
TerminalColors, TerminalColors,
TerminalHelper, TerminalHelper
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
help = """ """ help = """ """
# Specifies which files the loaderFilenames commands should target
default_filenames = [
"escrow_domain_contacts.daily.gov.GOV.txt",
"escrow_contacts.daily.gov.GOV.txt",
"escrow_domain_statuses.daily.gov.GOV.txt",
]
def add_arguments(self, parser): def add_arguments(self, parser):
""" """
OPTIONAL ARGUMENTS: OPTIONAL ARGUMENTS:
@ -79,34 +76,25 @@ class Command(BaseCommand):
--resetTable --resetTable
Used by the loaders to trigger a prompt for deleting all table entries. Used by the loaders to trigger a prompt for deleting all table entries.
Useful for testing purposes, but USE WITH CAUTION Useful for testing purposes, but USE WITH CAUTION
""" # noqa - line length """
parser.add_argument( parser.add_argument("--runLoaders",
"--runLoaders",
help="Runs all scripts (in sequence) for transition domain migrations", help="Runs all scripts (in sequence) for transition domain migrations",
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction)
)
parser.add_argument( parser.add_argument("--triggerLogins",
"--triggerLogins",
help="Simulates a user login for each user in domain invitation", help="Simulates a user login for each user in domain invitation",
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction)
)
# The following file arguments have default values for running in the sandbox # The following file arguments have default values for running in the sandbox
# For linter
script = "load_transition_domain migration script"
parser.add_argument( parser.add_argument(
"--loaderDirectory", "--loaderDirectory",
default="migrationdata", default="migrationData",
help=f"The location of the files used for the {script}", help="The location of the files used for load_transition_domain migration script"
) )
parser.add_argument( parser.add_argument(
"--loaderFilenames", "--loaderFilenames",
# For linter. default="escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt",
# Join all of the default filenames with a space.
# The resulting list will look like this: file1.txt file2.txt file3.txt
default=" ".join(self.default_filenames),
help="""The files used for load_transition_domain migration script. help="""The files used for load_transition_domain migration script.
Must appear IN ORDER and separated by spaces: Must appear IN ORDER and separated by spaces:
domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt domain_contacts_filename.txt contacts_filename.txt domain_statuses_filename.txt
@ -114,12 +102,10 @@ class Command(BaseCommand):
where... where...
- domain_contacts_filename is the Data file with domain contact information - domain_contacts_filename is the Data file with domain contact information
- contacts_filename is the Data file with contact information - contacts_filename is the Data file with contact information
- domain_statuses_filename is the Data file with domain status information""", # noqa - linter length - domain_statuses_filename is the Data file with domain status information"""
) )
parser.add_argument( parser.add_argument("--sep", default="|", help="Delimiter character for the loader files")
"--sep", default="|", help="Delimiter character for the loader files"
)
parser.add_argument("--debug", action=argparse.BooleanOptionalAction) parser.add_argument("--debug", action=argparse.BooleanOptionalAction)
@ -133,6 +119,8 @@ class Command(BaseCommand):
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction,
) )
def compare_tables(self, debug_on: bool): def compare_tables(self, debug_on: bool):
"""Does a diff between the transition_domain and the following tables: """Does a diff between the transition_domain and the following tables:
domain, domain_information and the domain_invitation. domain, domain_information and the domain_invitation.
@ -154,13 +142,13 @@ class Command(BaseCommand):
""" """
) )
# TODO: would filteredRelation be faster? #TODO: would filteredRelation be faster?
missing_domains = [] missing_domains = []
duplicate_domains = [] duplicate_domains = []
missing_domain_informations = [] missing_domain_informations = []
missing_domain_invites = [] missing_domain_invites = []
for transition_domain in TransitionDomain.objects.all(): # DEBUG: for transition_domain in TransitionDomain.objects.all():# DEBUG:
transition_domain_name = transition_domain.domain_name transition_domain_name = transition_domain.domain_name
transition_domain_email = transition_domain.username transition_domain_email = transition_domain.username
@ -172,38 +160,22 @@ class Command(BaseCommand):
# Check Domain table # Check Domain table
matching_domains = Domain.objects.filter(name=transition_domain_name) matching_domains = Domain.objects.filter(name=transition_domain_name)
# Check Domain Information table # Check Domain Information table
matching_domain_informations = DomainInformation.objects.filter( matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name)
domain__name=transition_domain_name
)
# Check Domain Invitation table # Check Domain Invitation table
matching_domain_invitations = DomainInvitation.objects.filter( matching_domain_invitations = DomainInvitation.objects.filter(email=transition_domain_email.lower(),
email=transition_domain_email.lower(), domain__name=transition_domain_name)
domain__name=transition_domain_name,
)
if len(matching_domains) == 0: if len(matching_domains) == 0:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""")
debug_on,
f"""{TerminalColors.YELLOW}Missing Domain{TerminalColors.ENDC}""", # noqa - line length
)
missing_domains.append(transition_domain_name) missing_domains.append(transition_domain_name)
elif len(matching_domains) > 1: elif len(matching_domains) > 1:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""")
debug_on,
f"""{TerminalColors.YELLOW}Duplicate Domain{TerminalColors.ENDC}""", # noqa - line length
)
duplicate_domains.append(transition_domain_name) duplicate_domains.append(transition_domain_name)
if len(matching_domain_informations) == 0: if len(matching_domain_informations) == 0:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""")
debug_on,
f"""{TerminalColors.YELLOW}Missing Domain Information{TerminalColors.ENDC}""", # noqa - line length
)
missing_domain_informations.append(transition_domain_name) missing_domain_informations.append(transition_domain_name)
if len(matching_domain_invitations) == 0: if len(matching_domain_invitations) == 0:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(debug_on, f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""")
debug_on,
f"""{TerminalColors.YELLOW}Missing Domain Invitation{TerminalColors.ENDC}""", # noqa - line length
)
missing_domain_invites.append(transition_domain_name) missing_domain_invites.append(transition_domain_name)
total_missing_domains = len(missing_domains) total_missing_domains = len(missing_domains)
@ -212,15 +184,9 @@ class Command(BaseCommand):
total_missing_domain_invitations = len(missing_domain_invites) total_missing_domain_invitations = len(missing_domain_invites)
missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains))) missing_domains_as_string = "{}".format(", ".join(map(str, missing_domains)))
duplicate_domains_as_string = "{}".format( duplicate_domains_as_string = "{}".format(", ".join(map(str, duplicate_domains)))
", ".join(map(str, duplicate_domains)) missing_domain_informations_as_string = "{}".format(", ".join(map(str, missing_domain_informations)))
) missing_domain_invites_as_string = "{}".format(", ".join(map(str, missing_domain_invites)))
missing_domain_informations_as_string = "{}".format(
", ".join(map(str, missing_domain_informations))
)
missing_domain_invites_as_string = "{}".format(
", ".join(map(str, missing_domain_invites))
)
logger.info( logger.info(
f"""{TerminalColors.OKGREEN} f"""{TerminalColors.OKGREEN}
@ -242,7 +208,7 @@ class Command(BaseCommand):
(These are transition domains which have no entires in the Domain Invitation Table) (These are transition domains which have no entires in the Domain Invitation Table)
{TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN} {TerminalColors.YELLOW}{missing_domain_invites_as_string}{TerminalColors.OKGREEN}
{TerminalColors.ENDC} {TerminalColors.ENDC}
""" # noqa - line length """
) )
def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool: def prompt_for_execution(self, command_string: str, prompt_title: str) -> bool:
@ -275,18 +241,15 @@ class Command(BaseCommand):
return True return True
def execute_command(self, command_string: str): def execute_command(self, command_string:str):
"""Executes the given command string""" """Executes the given command string"""
logger.info( logger.info(f"""{TerminalColors.OKCYAN}
f"""{TerminalColors.OKCYAN}
==== EXECUTING... ==== ==== EXECUTING... ====
{TerminalColors.ENDC}""" {TerminalColors.ENDC}""")
)
os.system(f"{command_string}") os.system(f"{command_string}")
def run_load_transition_domain_script( def run_load_transition_domain_script(self,
self,
file_location: str, file_location: str,
domain_contacts_filename: str, domain_contacts_filename: str,
contacts_filename: str, contacts_filename: str,
@ -295,14 +258,13 @@ class Command(BaseCommand):
reset_table: bool, reset_table: bool,
debug_on: bool, debug_on: bool,
prompts_enabled: bool, prompts_enabled: bool,
debug_max_entries_to_parse: int, debug_max_entries_to_parse: int):
):
"""Runs the load_transition_domain script""" """Runs the load_transition_domain script"""
# Create the command string # Create the command string
command_string = "./manage.py load_transition_domain " command_string = "./manage.py load_transition_domain "
command_string += file_location + domain_contacts_filename + " " command_string += file_location+domain_contacts_filename + " "
command_string += file_location + contacts_filename + " " command_string += file_location+contacts_filename + " "
command_string += file_location + domain_statuses_filename + " " command_string += file_location+domain_statuses_filename + " "
if sep is not None and sep != "|": if sep is not None and sep != "|":
command_string += f"--sep {sep} " command_string += f"--sep {sep} "
if reset_table: if reset_table:
@ -314,13 +276,12 @@ class Command(BaseCommand):
# Execute the command string # Execute the command string
if prompts_enabled: if prompts_enabled:
self.prompt_for_execution( self.prompt_for_execution(command_string, "Running load_transition_domain script")
command_string, "Running load_transition_domain script"
)
return return
self.execute_command(command_string) self.execute_command(command_string)
def run_transfer_script(self, debug_on: bool, prompts_enabled: bool):
def run_transfer_script(self, debug_on:bool, prompts_enabled: bool):
"""Runs the transfer_transition_domains_to_domains script""" """Runs the transfer_transition_domains_to_domains script"""
# Create the command string # Create the command string
command_string = "./manage.py transfer_transition_domains_to_domains " command_string = "./manage.py transfer_transition_domains_to_domains "
@ -328,25 +289,25 @@ class Command(BaseCommand):
command_string += "--debug " command_string += "--debug "
# Execute the command string # Execute the command string
if prompts_enabled: if prompts_enabled:
self.prompt_for_execution( self.prompt_for_execution(command_string, "Running transfer_transition_domains_to_domains script")
command_string, "Running transfer_transition_domains_to_domains script"
)
return return
self.execute_command(command_string) self.execute_command(command_string)
def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool): def run_send_invites_script(self, debug_on: bool, prompts_enabled: bool):
"""Runs the send_domain_invitations script""" """Runs the send_domain_invitations script"""
# Create the command string... # Create the command string...
command_string = "./manage.py send_domain_invitations -s" command_string = "./manage.py send_domain_invitations -s"
# Execute the command string # Execute the command string
if prompts_enabled: if prompts_enabled:
self.prompt_for_execution( self.prompt_for_execution(command_string, "Running send_domain_invitations script")
command_string, "Running send_domain_invitations script"
)
return return
self.execute_command(command_string) self.execute_command(command_string)
def run_migration_scripts(self, prompts_enabled: bool, options):
def run_migration_scripts(self,
prompts_enabled: bool,
options):
"""Runs the following migration scripts (in order): """Runs the following migration scripts (in order):
1 - imports for trans domains 1 - imports for trans domains
2 - transfer to domain & domain invitation""" 2 - transfer to domain & domain invitation"""
@ -355,15 +316,16 @@ class Command(BaseCommand):
sep = options.get("sep") sep = options.get("sep")
reset_table = options.get("resetTable") reset_table = options.get("resetTable")
debug_on = options.get("debug") debug_on = options.get("debug")
debug_max_entries_to_parse = int(options.get("limitParse")) debug_max_entries_to_parse = int(
options.get("limitParse")
)
# Grab filepath information from the arguments # Grab filepath information from the arguments
file_location = options.get("loaderDirectory") + "/" file_location = options.get("loaderDirectory")+"/"
filenames = options.get("loaderFilenames").split() filenames = options.get("loaderFilenames").split(",")
if len(filenames) < 3: if len(filenames) < 3:
filenames_as_string = "{}".format(", ".join(map(str, filenames))) filenames_as_string = "{}".format(", ".join(map(str, filenames)))
logger.info( logger.info(f"""
f"""
{TerminalColors.FAIL} {TerminalColors.FAIL}
--loaderFilenames expected 3 filenames to follow it, --loaderFilenames expected 3 filenames to follow it,
but only {len(filenames)} were given: but only {len(filenames)} were given:
@ -372,8 +334,7 @@ class Command(BaseCommand):
PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN PLEASE MODIFY THE SCRIPT AND TRY RUNNING IT AGAIN
============= TERMINATING ============= ============= TERMINATING =============
{TerminalColors.ENDC} {TerminalColors.ENDC}
""" """)
)
sys.exit() sys.exit()
domain_contacts_filename = filenames[0] domain_contacts_filename = filenames[0]
contacts_filename = filenames[1] contacts_filename = filenames[1]
@ -405,21 +366,18 @@ class Command(BaseCommand):
# correct file inputs in their original command # correct file inputs in their original command
# prompt and exit this subroutine # prompt and exit this subroutine
if not files_are_correct: if not files_are_correct:
logger.info( logger.info(f"""
f"""
{TerminalColors.YELLOW} {TerminalColors.YELLOW}
PLEASE Re-Run the script with the correct file location and filenames: PLEASE Re-Run the script with the correct file location and filenames:
EXAMPLE: EXAMPLE:
docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt docker compose run -T app ./manage.py test_domain_migration --runLoaders --loaderDirectory /app/tmp --loaderFilenames escrow_domain_contacts.daily.gov.GOV.txt escrow_contacts.daily.gov.GOV.txt escrow_domain_statuses.daily.gov.GOV.txt
""" # noqa - line length """)
)
return return
# Proceed executing the migration scripts # Proceed executing the migration scripts
self.run_load_transition_domain_script( self.run_load_transition_domain_script(file_location,
file_location,
domain_contacts_filename, domain_contacts_filename,
contacts_filename, contacts_filename,
domain_statuses_filename, domain_statuses_filename,
@ -427,22 +385,21 @@ class Command(BaseCommand):
reset_table, reset_table,
debug_on, debug_on,
prompts_enabled, prompts_enabled,
debug_max_entries_to_parse, debug_max_entries_to_parse)
)
self.run_transfer_script(debug_on, prompts_enabled) self.run_transfer_script(debug_on, prompts_enabled)
def simulate_user_logins(self, debug_on): def simulate_user_logins(self, debug_on):
"""Simulates logins for users (this will add """Simulates logins for users (this will add
Domain Information objects to our tables)""" Domain Information objects to our tables)"""
logger.info( logger.info(f""
f""
f"{TerminalColors.OKCYAN}" f"{TerminalColors.OKCYAN}"
f"================== SIMULATING LOGINS ==================" f"================== SIMULATING LOGINS =================="
f"{TerminalColors.ENDC}" f"{TerminalColors.ENDC}")
)
# command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins"
command_string = "python ./manage.py test registrar.tests.test_transition_domain_migrations_wiuth_logins.TestLogins.test_user_logins"
# for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff # for invite in DomainInvitation.objects.all(): #TODO: limit to our stuff
# #DEBUG: # #DEBUG:
@ -462,6 +419,7 @@ class Command(BaseCommand):
# logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""") # logger.info(f"""{TerminalColors.YELLOW}(Deleting temporary user object){TerminalColors.ENDC}""")
# user.delete() # user.delete()
def handle( def handle(
self, self,
**options, **options,
@ -484,7 +442,7 @@ class Command(BaseCommand):
# the terminal so the user knows what is # the terminal so the user knows what is
# enabled. # enabled.
debug_on = options.get("debug") debug_on = options.get("debug")
prompts_enabled = debug_on # TODO: add as argument? prompts_enabled = debug_on #TODO: add as argument?
run_loaders_enabled = options.get("runLoaders") run_loaders_enabled = options.get("runLoaders")
simulate_user_login_enabled = options.get("triggerLogins") simulate_user_login_enabled = options.get("triggerLogins")
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
@ -493,7 +451,7 @@ class Command(BaseCommand):
----------DEBUG MODE ON---------- ----------DEBUG MODE ON----------
Detailed print statements activated. Detailed print statements activated.
{TerminalColors.ENDC} {TerminalColors.ENDC}
""", """
) )
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
run_loaders_enabled, run_loaders_enabled,
@ -502,7 +460,7 @@ class Command(BaseCommand):
All migration scripts will be run before All migration scripts will be run before
analyzing the data. analyzing the data.
{TerminalColors.ENDC} {TerminalColors.ENDC}
""", """
) )
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
run_loaders_enabled, run_loaders_enabled,
@ -510,7 +468,7 @@ class Command(BaseCommand):
----------TRIGGER LOGINS ON---------- ----------TRIGGER LOGINS ON----------
Will be simulating user logins Will be simulating user logins
{TerminalColors.ENDC} {TerminalColors.ENDC}
""", """
) )
# If a user decides to run all migration # If a user decides to run all migration
# scripts, they may or may not wish to # scripts, they may or may not wish to
@ -526,7 +484,7 @@ class Command(BaseCommand):
# STEP 1 -- RUN LOADERS # STEP 1 -- RUN LOADERS
# Run migration scripts if specified by user # Run migration scripts if specified by user
if run_loaders_enabled: if run_loaders_enabled:
self.run_migration_scripts(options, prompts_enabled) self.run_migration_scripts(prompts_enabled, options)
prompt_continuation_of_analysis = True prompt_continuation_of_analysis = True
# STEP 2 -- SIMULATE LOGINS # STEP 2 -- SIMULATE LOGINS
@ -539,17 +497,18 @@ class Command(BaseCommand):
# automatically execute this as the final step # automatically execute this as the final step
# to ensure Domain Information objects get added # to ensure Domain Information objects get added
# to the database.) # to the database.)
if run_loaders_enabled:
if prompts_enabled: # if run_loaders_enabled:
simulate_user_login_enabled = TerminalHelper.query_yes_no( # if prompts_enabled:
f"""{TerminalColors.FAIL} # simulate_user_login_enabled = TerminalHelper.query_yes_no(
Proceed with simulating user logins? # f"""{TerminalColors.FAIL}
{TerminalColors.ENDC}""" # Proceed with simulating user logins?
) # {TerminalColors.ENDC}"""
if not simulate_user_login_enabled: # )
return # if not simulate_user_login_enabled:
self.simulate_user_logins(debug_on) # return
prompt_continuation_of_analysis = True # self.simulate_user_logins(debug_on)
# prompt_continuation_of_analysis = True
# STEP 3 -- SEND INVITES # STEP 3 -- SEND INVITES
if prompts_enabled: if prompts_enabled:

8
src/registrar/tests/test_transition_domain_migrations_with_logins.py → src/registrar/tests/test_transition_domain_migrations.py
View file

@ -23,6 +23,14 @@ class TestLogins(TestCase):
# clean out the roles each time # clean out the roles each time
UserDomainRole.objects.all().delete() UserDomainRole.objects.all().delete()
def test_migration_functions(self):
""" Run the master migration script using local test data """
""" (analyze the tables just like the migration script does, but add assert statements) """
#TODO: finish me!
self.assertTrue(True)
def test_user_logins(self): def test_user_logins(self):
"""A new user's first_login callback retrieves their invitations.""" """A new user's first_login callback retrieves their invitations."""
self.user.first_login() self.user.first_login()