From c68a8f30d0d8bc6e5587ecf3a665bef71bcab1a3 Mon Sep 17 00:00:00 2001 From: Katherine-Osos <119689946+Katherine-Osos@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:25:54 -0600 Subject: [PATCH 01/78] Dashboard: adjust placeholder messaging --- src/registrar/templates/home.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/home.html b/src/registrar/templates/home.html index 0bc792cef..8e056924f 100644 --- a/src/registrar/templates/home.html +++ b/src/registrar/templates/home.html @@ -138,7 +138,7 @@ aria-live="polite" > {% else %} -

You don't have any active domain requests right now

+

You don't have any domain requests yet

{% endif %} From caed611f21d374b5426d655ff8173df9888a2ebf Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:42:01 -0700 Subject: [PATCH 02/78] Create extend_expiration_dates.py --- .../commands/extend_expiration_dates.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/registrar/management/commands/extend_expiration_dates.py diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py new file mode 100644 index 000000000..d098fdf1c --- /dev/null +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -0,0 +1,102 @@ +"""Data migration: Extends expiration dates for valid domains""" + +import argparse +from datetime import date +import logging + +from django.core.management import BaseCommand +from registrar.models import Domain +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Extends expiration dates for valid domains" + + # Generates test transition domains for testing send_domain_invitations script. + # Running this script removes all existing transition domains, so use with caution. + # Transition domains are created with email addresses provided as command line + # argument. Email addresses for testing are passed as comma delimited list of + # email addresses, and are required to be provided. Email addresses from the list + # are assigned to transition domains at time of creation. + + def add_arguments(self, parser): + """Add command line arguments.""" + parser.add_argument( + "--extensionAmount", + type=int, + default=1, + help="Determines the period (in years) to extend expiration dates by", + ) + parser.add_argument( + "--parseLimit", + type=int, + default=0, + help="Sets a cap on the number of records to parse", + ) + + def handle(self, **options): + """""" + extension_amount = options.get("extensionAmount") + parse_limit = options.get("parseLimit") + + # Does a check to see if parse_limit is a positive int + self.check_if_positive_int(parse_limit, "parseLimit") + + # TODO - Do we need to check status? + valid_domains = Domain.objects.filter( + expiration_date__gte=date(2023, 11, 15), + State=Domain.State.READY + ) + + TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + info_to_inspect=f""" + ==Extension Amount== + Period: {extension_amount} year(s) + + ==Proposed Changes== + Domains to change: {valid_domains.count()} + """, + prompt_title="Do you wish to modify Expiration Dates for the given Domains?", + ) + + logger.info( + f"{TerminalColors.MAGENTA}" + "Preparing to extend expiration dates..." + f"{TerminalColors.ENDC}" + ) + + for i, domain in enumerate(valid_domains): + if i > parse_limit: + break + + try: + domain.renew_domain(extension_amount) + except Exception as err: + logger.error( + f"{TerminalColors.OKBLUE}" + f"Failed to update expiration date for {domain}" + f"{TerminalColors.ENDC}" + ) + raise err + else: + logger.info( + f"{TerminalColors.OKBLUE}" + f"Successfully updated expiration date for {domain}" + f"{TerminalColors.ENDC}" + ) + + def check_if_positive_int(self, value: int, var_name: str): + """ + Determines if the given integer value is postive or not. + If not, it raises an ArgumentTypeError + """ + if value < 0: + raise argparse.ArgumentTypeError( + f"{value} is an invalid integer value for {var_name}. " + "Must be positive." + ) + + return value From 8faef8b9d19f36a786b044f96f27676e22824bc7 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Dec 2023 13:25:56 -0700 Subject: [PATCH 03/78] Finish script --- .../commands/extend_expiration_dates.py | 201 +++++++++++++++--- 1 file changed, 167 insertions(+), 34 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index d098fdf1c..e9bcc2ff4 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -5,8 +5,13 @@ from datetime import date import logging from django.core.management import BaseCommand +from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper +from dateutil.relativedelta import relativedelta + from epplib.exceptions import TransportError +except ImportError: + pass logger = logging.getLogger(__name__) @@ -14,12 +19,13 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Extends expiration dates for valid domains" - # Generates test transition domains for testing send_domain_invitations script. - # Running this script removes all existing transition domains, so use with caution. - # Transition domains are created with email addresses provided as command line - # argument. Email addresses for testing are passed as comma delimited list of - # email addresses, and are required to be provided. Email addresses from the list - # are assigned to transition domains at time of creation. + def __init__(self): + """Sets global variables for code tidyness""" + super().__init__() + self.update_success = [] + self.update_skipped = [] + self.update_failed = [] + self.debug = False def add_arguments(self, parser): """Add command line arguments.""" @@ -30,26 +36,98 @@ class Command(BaseCommand): help="Determines the period (in years) to extend expiration dates by", ) parser.add_argument( - "--parseLimit", + "--limitParse", type=int, default=0, help="Sets a cap on the number of records to parse", ) + parser.add_argument( + "--disableIdempotentCheck", + action=argparse.BooleanOptionalAction, + help="Disable script idempotence" + ) + parser.add_argument( + "--debug", + action=argparse.BooleanOptionalAction, + help="Increases log chattiness" + ) def handle(self, **options): """""" extension_amount = options.get("extensionAmount") - parse_limit = options.get("parseLimit") + limit_parse = options.get("limitParse") + disable_idempotence = options.get("disableIdempotentCheck") + self.debug = options.get("debug") - # Does a check to see if parse_limit is a positive int - self.check_if_positive_int(parse_limit, "parseLimit") + # Does a check to see if parse_limit is a positive int. + # Raise an error if not. + self.check_if_positive_int(limit_parse, "limitParse") - # TODO - Do we need to check status? valid_domains = Domain.objects.filter( expiration_date__gte=date(2023, 11, 15), - State=Domain.State.READY - ) + state=Domain.State.READY + ).order_by("name") + domains_to_change_count = valid_domains.count() + if limit_parse != 0 and limit_parse < domains_to_change_count: + domains_to_change_count = limit_parse + + # Determines if we should continue code execution or not. + # If the user prompts 'N', a sys.exit() will be called. + self.prompt_user_to_proceed(extension_amount, domains_to_change_count) + + for i, domain in enumerate(valid_domains): + if limit_parse != 0 and i > limit_parse: + break + + is_idempotent = self.idempotence_check(domain, extension_amount) + if not disable_idempotence and not is_idempotent: + self.update_skipped.append(domain.name) + else: + self.extend_expiration_date_on_domain(domain, extension_amount) + + self.log_script_run_summary() + + def extend_expiration_date_on_domain(self, domain: Domain, extension_amount: int): + """ + Given a particular domain, + extend the expiration date by the period specified in extension_amount + """ + try: + domain.renew_domain(extension_amount) + except (RegistryError, TransportError) as err: + logger.error( + f"{TerminalColors.FAIL}" + f"Failed to update expiration date for {domain}" + f"{TerminalColors.ENDC}" + ) + logger.error(err) + except Exception as err: + self.log_script_run_summary() + raise err + else: + self.update_success.append(domain.name) + logger.info( + f"{TerminalColors.OKCYAN}" + f"Successfully updated expiration date for {domain}" + f"{TerminalColors.ENDC}" + ) + + # == Helper functions == # + def idempotence_check(self, domain, extension_amount): + """Determines if the proposed operation violates idempotency""" + proposed_date = domain.expiration_date + relativedelta(years=extension_amount) + # Because our migration data had a hard stop date, we can determine if our change + # is valid simply checking if adding a year to our current date yields a greater date + # than the proposed. + # CAVEAT: This check stops working after a year has elapsed between when this script + # was ran, and when it was ran again. This is good enough for now, but a more robust + # solution would be a DB flag. + is_idempotent = proposed_date < date.today() + relativedelta(years=extension_amount) + return is_idempotent + + def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): + """Asks if the user wants to proceed with this action""" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, info_to_inspect=f""" @@ -57,7 +135,7 @@ class Command(BaseCommand): Period: {extension_amount} year(s) ==Proposed Changes== - Domains to change: {valid_domains.count()} + Domains to change: {domains_to_change_count} """, prompt_title="Do you wish to modify Expiration Dates for the given Domains?", ) @@ -68,29 +146,84 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}" ) - for i, domain in enumerate(valid_domains): - if i > parse_limit: - break + def log_script_run_summary(self): + """Prints success, failed, and skipped counts, as well as + all affected domains.""" + update_success_count = len(self.update_success) + update_failed_count = len(self.update_failed) + update_skipped_count = len(self.update_skipped) + if update_failed_count == 0 and update_skipped_count == 0: + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED =============== + Updated {update_success_count} Domain entries + {TerminalColors.ENDC} + """ + ) + TerminalHelper.print_conditional( + self.debug, + f""" + {TerminalColors.OKGREEN} + Updated the following Domains: {self.update_success} + {TerminalColors.ENDC} + """, + ) + elif update_failed_count == 0: + TerminalHelper.print_conditional( + self.debug, + f""" + {TerminalColors.OKGREEN} + Updated the following Domains: {self.update_success} + {TerminalColors.ENDC} + + {TerminalColors.YELLOW} + Skipped the following Domains: {self.update_skipped} + {TerminalColors.ENDC} + """, + ) + logger.info( + f"""{TerminalColors.YELLOW} + ============= FINISHED =============== + Updated {update_success_count} Domain entries + + ----- IDEMPOTENCY CHECK FAILED ----- + Skipped updating {update_skipped_count} Domain entries + {TerminalColors.ENDC} + """ + ) + else: + TerminalHelper.print_conditional( + self.debug, + f""" + {TerminalColors.OKGREEN} + Updated the following Domains: {self.update_success} + {TerminalColors.ENDC} + + {TerminalColors.YELLOW} + Skipped the following Domains: {self.update_skipped} + {TerminalColors.ENDC} + + {TerminalColors.FAIL} + Failed to update the following Domains: {self.update_failed} + {TerminalColors.ENDC} + """, + ) + logger.info( + f"""{TerminalColors.FAIL} + ============= FINISHED =============== + Updated {update_success_count} Domain entries + + ----- UPDATE FAILED ----- + Failed to update {update_failed_count} Domain entries, + Skipped updating {update_skipped_count} Domain entries + {TerminalColors.ENDC} + """ + ) - try: - domain.renew_domain(extension_amount) - except Exception as err: - logger.error( - f"{TerminalColors.OKBLUE}" - f"Failed to update expiration date for {domain}" - f"{TerminalColors.ENDC}" - ) - raise err - else: - logger.info( - f"{TerminalColors.OKBLUE}" - f"Successfully updated expiration date for {domain}" - f"{TerminalColors.ENDC}" - ) def check_if_positive_int(self, value: int, var_name: str): """ - Determines if the given integer value is postive or not. + Determines if the given integer value is positive or not. If not, it raises an ArgumentTypeError """ if value < 0: @@ -99,4 +232,4 @@ class Command(BaseCommand): "Must be positive." ) - return value + return value \ No newline at end of file From 6155d225e6f1a70e407adfb56b2a089b05d68b46 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:41:27 -0700 Subject: [PATCH 04/78] Add debug, logging, and test skeleton --- .../commands/extend_expiration_dates.py | 69 +++++++++++-------- .../test_transition_domain_migrations.py | 64 +++++++++++++++++ 2 files changed, 106 insertions(+), 27 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index e9bcc2ff4..715313b91 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -9,6 +9,7 @@ from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from dateutil.relativedelta import relativedelta +try: from epplib.exceptions import TransportError except ImportError: pass @@ -25,7 +26,6 @@ class Command(BaseCommand): self.update_success = [] self.update_skipped = [] self.update_failed = [] - self.debug = False def add_arguments(self, parser): """Add command line arguments.""" @@ -53,11 +53,27 @@ class Command(BaseCommand): ) def handle(self, **options): - """""" + """ + Extends the expiration dates for valid domains. + + It first retrieves the command line options and checks if the parse limit is a positive integer. + Then, it fetches the valid domains from the database and calculates the number of domains to change. + If a parse limit is set and it's less than the total number of valid domains, + the number of domains to change is set to the parse limit. + + For each domain, it checks if the operation is idempotent. + If the idempotence check is not disabled and the operation is not idempotent, the domain is skipped. + Otherwise, the expiration date of the domain is extended. + + Finally, it logs a summary of the script run, + including the number of successful, failed, and skipped updates. + """ + + # Retrieve command line options extension_amount = options.get("extensionAmount") limit_parse = options.get("limitParse") disable_idempotence = options.get("disableIdempotentCheck") - self.debug = options.get("debug") + debug = options.get("debug") # Does a check to see if parse_limit is a positive int. # Raise an error if not. @@ -84,11 +100,11 @@ class Command(BaseCommand): if not disable_idempotence and not is_idempotent: self.update_skipped.append(domain.name) else: - self.extend_expiration_date_on_domain(domain, extension_amount) + self.extend_expiration_date_on_domain(domain, extension_amount, debug) - self.log_script_run_summary() + self.log_script_run_summary(debug) - def extend_expiration_date_on_domain(self, domain: Domain, extension_amount: int): + def extend_expiration_date_on_domain(self, domain: Domain, extension_amount: int, debug: bool): """ Given a particular domain, extend the expiration date by the period specified in extension_amount @@ -103,7 +119,7 @@ class Command(BaseCommand): ) logger.error(err) except Exception as err: - self.log_script_run_summary() + self.log_script_run_summary(debug) raise err else: self.update_success.append(domain.name) @@ -123,7 +139,7 @@ class Command(BaseCommand): # CAVEAT: This check stops working after a year has elapsed between when this script # was ran, and when it was ran again. This is good enough for now, but a more robust # solution would be a DB flag. - is_idempotent = proposed_date < date.today() + relativedelta(years=extension_amount) + is_idempotent = proposed_date < (date.today() + relativedelta(years=extension_amount+1)) return is_idempotent def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): @@ -146,7 +162,20 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}" ) - def log_script_run_summary(self): + def check_if_positive_int(self, value: int, var_name: str): + """ + Determines if the given integer value is positive or not. + If not, it raises an ArgumentTypeError + """ + if value < 0: + raise argparse.ArgumentTypeError( + f"{value} is an invalid integer value for {var_name}. " + "Must be positive." + ) + + return value + + def log_script_run_summary(self, debug): """Prints success, failed, and skipped counts, as well as all affected domains.""" update_success_count = len(self.update_success) @@ -161,7 +190,7 @@ class Command(BaseCommand): """ ) TerminalHelper.print_conditional( - self.debug, + debug, f""" {TerminalColors.OKGREEN} Updated the following Domains: {self.update_success} @@ -170,7 +199,7 @@ class Command(BaseCommand): ) elif update_failed_count == 0: TerminalHelper.print_conditional( - self.debug, + debug, f""" {TerminalColors.OKGREEN} Updated the following Domains: {self.update_success} @@ -193,7 +222,7 @@ class Command(BaseCommand): ) else: TerminalHelper.print_conditional( - self.debug, + debug, f""" {TerminalColors.OKGREEN} Updated the following Domains: {self.update_success} @@ -218,18 +247,4 @@ class Command(BaseCommand): Skipped updating {update_skipped_count} Domain entries {TerminalColors.ENDC} """ - ) - - - def check_if_positive_int(self, value: int, var_name: str): - """ - Determines if the given integer value is positive or not. - If not, it raises an ArgumentTypeError - """ - if value < 0: - raise argparse.ArgumentTypeError( - f"{value} is an invalid integer value for {var_name}. " - "Must be positive." - ) - - return value \ No newline at end of file + ) \ No newline at end of file diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4e549bdd6..0fd36a2d4 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -21,6 +21,70 @@ from registrar.models.contact import Contact from .common import less_console_noise +class TestExtendExpirationDates(TestCase): + def setUp(self): + """Defines the file name of migration_json and the folder its contained in""" + self.test_data_file_location = "registrar/tests/data" + self.migration_json_filename = "test_migrationFilepaths.json" + + def tearDown(self): + """Deletes all DB objects related to migrations""" + # Delete domain information + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainInvitation.objects.all().delete() + TransitionDomain.objects.all().delete() + + # Delete users + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def run_load_domains(self): + """ + This method executes the load_transition_domain command. + + It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, + which is a user prompt in the terminal. The mock function always returns True, + allowing the test to proceed without manual user input. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_transition_domain", + self.migration_json_filename, + directory=self.test_data_file_location, + ) + + def run_transfer_domains(self): + """ + This method executes the transfer_transition_domains_to_domains command. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command("transfer_transition_domains_to_domains") + + def run_extend_expiration_dates(self): + """ + This method executes the transfer_transition_domains_to_domains command. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + call_command("extend_expiration_dates") + + def test_extends_correctly(self): + pass + class TestOrganizationMigration(TestCase): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" From 93b5cee23d4a9f9c2f52fd9e44ecba5f769f72ef Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:35:53 -0700 Subject: [PATCH 05/78] Update test_transition_domain_migrations.py --- .../test_transition_domain_migrations.py | 97 +++++++++++-------- 1 file changed, 56 insertions(+), 41 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 0fd36a2d4..b8301c6b1 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -18,17 +18,23 @@ from unittest.mock import patch from registrar.models.contact import Contact -from .common import less_console_noise +from .common import MockEppLib, less_console_noise +from dateutil.relativedelta import relativedelta -class TestExtendExpirationDates(TestCase): +class TestExtendExpirationDates(MockEppLib): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" - self.test_data_file_location = "registrar/tests/data" - self.migration_json_filename = "test_migrationFilepaths.json" + super().setUp() + self.domain, _ = Domain.objects.get_or_create( + name="fake.gov", + state=Domain.State.READY, + expiration_date=datetime.date(2023, 5, 25) + ) def tearDown(self): """Deletes all DB objects related to migrations""" + super().tearDown() # Delete domain information Domain.objects.all().delete() DomainInformation.objects.all().delete() @@ -38,40 +44,6 @@ class TestExtendExpirationDates(TestCase): # Delete users User.objects.all().delete() UserDomainRole.objects.all().delete() - - def run_load_domains(self): - """ - This method executes the load_transition_domain command. - - It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, - which is a user prompt in the terminal. The mock function always returns True, - allowing the test to proceed without manual user input. - - The 'call_command' function from Django's management framework is then used to - execute the load_transition_domain command with the specified arguments. - """ - with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa - return_value=True, - ): - call_command( - "load_transition_domain", - self.migration_json_filename, - directory=self.test_data_file_location, - ) - - def run_transfer_domains(self): - """ - This method executes the transfer_transition_domains_to_domains command. - - The 'call_command' function from Django's management framework is then used to - execute the load_transition_domain command with the specified arguments. - """ - with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa - return_value=True, - ): - call_command("transfer_transition_domains_to_domains") def run_extend_expiration_dates(self): """ @@ -80,10 +52,53 @@ class TestExtendExpirationDates(TestCase): The 'call_command' function from Django's management framework is then used to execute the load_transition_domain command with the specified arguments. """ - call_command("extend_expiration_dates") + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command("extend_expiration_dates") - def test_extends_correctly(self): - pass + def test_extends_expiration_date_correctly(self): + desired_domain = Domain.objects.filter(name="fake.gov").get() + desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + + # Run the expiration date script + self.run_extend_expiration_dates() + + self.assertEqual(desired_domain, self.domain) + + # Explicitly test the expiration date + self.assertEqual(self.domain.expiration_date, datetime.date(2024, 5, 25)) + + # TODO ALSO NEED A TEST FOR NON READY DOMAINS + def test_extends_expiration_date_skips_non_current(self): + desired_domain = Domain.objects.filter(name="fake.gov").get() + desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + + # Run the expiration date script + self.run_extend_expiration_dates() + + current_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date. The extend_expiration_dates script + # will skip all dates less than date(2023, 11, 15), meaning that this domain + # should not be affected by the change. + self.assertEqual(current_domain.expiration_date, datetime.date(2023, 5, 25)) + + def test_extends_expiration_date_idempotent(self): + desired_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + + # Run the expiration date script + self.run_extend_expiration_dates() + + current_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date + self.assertEqual(desired_domain.expiration_date, datetime.date(2023, 9, 30)) + class TestOrganizationMigration(TestCase): def setUp(self): From ae6cea710043b78a1b3fef9941f2c8b71de7392d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:19:11 -0700 Subject: [PATCH 06/78] Add unit tests --- .../commands/extend_expiration_dates.py | 51 ++++------- src/registrar/tests/common.py | 20 +++++ .../test_transition_domain_migrations.py | 90 ++++++++++++++----- 3 files changed, 105 insertions(+), 56 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 715313b91..fe25da504 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -9,6 +9,7 @@ from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from dateutil.relativedelta import relativedelta + try: from epplib.exceptions import TransportError except ImportError: @@ -42,15 +43,9 @@ class Command(BaseCommand): help="Sets a cap on the number of records to parse", ) parser.add_argument( - "--disableIdempotentCheck", - action=argparse.BooleanOptionalAction, - help="Disable script idempotence" - ) - parser.add_argument( - "--debug", - action=argparse.BooleanOptionalAction, - help="Increases log chattiness" + "--disableIdempotentCheck", action=argparse.BooleanOptionalAction, help="Disable script idempotence" ) + parser.add_argument("--debug", action=argparse.BooleanOptionalAction, help="Increases log chattiness") def handle(self, **options): """ @@ -58,14 +53,14 @@ class Command(BaseCommand): It first retrieves the command line options and checks if the parse limit is a positive integer. Then, it fetches the valid domains from the database and calculates the number of domains to change. - If a parse limit is set and it's less than the total number of valid domains, + If a parse limit is set and it's less than the total number of valid domains, the number of domains to change is set to the parse limit. - For each domain, it checks if the operation is idempotent. + For each domain, it checks if the operation is idempotent. If the idempotence check is not disabled and the operation is not idempotent, the domain is skipped. Otherwise, the expiration date of the domain is extended. - Finally, it logs a summary of the script run, + Finally, it logs a summary of the script run, including the number of successful, failed, and skipped updates. """ @@ -80,8 +75,7 @@ class Command(BaseCommand): self.check_if_positive_int(limit_parse, "limitParse") valid_domains = Domain.objects.filter( - expiration_date__gte=date(2023, 11, 15), - state=Domain.State.READY + expiration_date__gte=date(2023, 11, 15), state=Domain.State.READY ).order_by("name") domains_to_change_count = valid_domains.count() @@ -95,15 +89,15 @@ class Command(BaseCommand): for i, domain in enumerate(valid_domains): if limit_parse != 0 and i > limit_parse: break - + is_idempotent = self.idempotence_check(domain, extension_amount) if not disable_idempotence and not is_idempotent: self.update_skipped.append(domain.name) else: self.extend_expiration_date_on_domain(domain, extension_amount, debug) - + self.log_script_run_summary(debug) - + def extend_expiration_date_on_domain(self, domain: Domain, extension_amount: int, debug: bool): """ Given a particular domain, @@ -113,9 +107,7 @@ class Command(BaseCommand): domain.renew_domain(extension_amount) except (RegistryError, TransportError) as err: logger.error( - f"{TerminalColors.FAIL}" - f"Failed to update expiration date for {domain}" - f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}" f"Failed to update expiration date for {domain}" f"{TerminalColors.ENDC}" ) logger.error(err) except Exception as err: @@ -124,9 +116,7 @@ class Command(BaseCommand): else: self.update_success.append(domain.name) logger.info( - f"{TerminalColors.OKCYAN}" - f"Successfully updated expiration date for {domain}" - f"{TerminalColors.ENDC}" + f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" ) # == Helper functions == # @@ -139,7 +129,7 @@ class Command(BaseCommand): # CAVEAT: This check stops working after a year has elapsed between when this script # was ran, and when it was ran again. This is good enough for now, but a more robust # solution would be a DB flag. - is_idempotent = proposed_date < (date.today() + relativedelta(years=extension_amount+1)) + is_idempotent = proposed_date < (date.today() + relativedelta(years=extension_amount + 1)) return is_idempotent def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): @@ -156,27 +146,22 @@ class Command(BaseCommand): prompt_title="Do you wish to modify Expiration Dates for the given Domains?", ) - logger.info( - f"{TerminalColors.MAGENTA}" - "Preparing to extend expiration dates..." - f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.MAGENTA}" "Preparing to extend expiration dates..." f"{TerminalColors.ENDC}") def check_if_positive_int(self, value: int, var_name: str): """ - Determines if the given integer value is positive or not. + Determines if the given integer value is positive or not. If not, it raises an ArgumentTypeError """ if value < 0: raise argparse.ArgumentTypeError( - f"{value} is an invalid integer value for {var_name}. " - "Must be positive." + f"{value} is an invalid integer value for {var_name}. " "Must be positive." ) return value def log_script_run_summary(self, debug): - """Prints success, failed, and skipped counts, as well as + """Prints success, failed, and skipped counts, as well as all affected domains.""" update_success_count = len(self.update_success) update_failed_count = len(self.update_failed) @@ -247,4 +232,4 @@ class Command(BaseCommand): Skipped updating {update_skipped_count} Domain entries {TerminalColors.ENDC} """ - ) \ No newline at end of file + ) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index f17e0f9fa..5899ce78a 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -819,6 +819,16 @@ class MockEppLib(TestCase): ex_date=datetime.date(2023, 5, 25), ) + mockDnsNeededRenewedDomainExpDate = fakedEppObject( + "fakeneeded.gov", + ex_date=datetime.date(2023, 2, 15), + ) + + mockRecentRenewedDomainExpDate = fakedEppObject( + "waterbutpurple.gov", + ex_date=datetime.date(2025, 1, 10), + ) + def _mockDomainName(self, _name, _avail=False): return MagicMock( res_data=[ @@ -919,6 +929,16 @@ class MockEppLib(TestCase): def mockRenewDomainCommand(self, _request, cleaned): if getattr(_request, "name", None) == "fake-error.gov": raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) + elif getattr(_request, "name", None) == "waterbutpurple.gov": + return MagicMock( + res_data=[self.mockRecentRenewedDomainExpDate], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) + elif getattr(_request, "name", None) == "fakeneeded.gov": + return MagicMock( + res_data=[self.mockDnsNeededRenewedDomainExpDate], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) else: return MagicMock( res_data=[self.mockRenewedDomainExpDate], diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index b8301c6b1..4fbbcbfc8 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -26,10 +26,14 @@ class TestExtendExpirationDates(MockEppLib): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" super().setUp() - self.domain, _ = Domain.objects.get_or_create( - name="fake.gov", - state=Domain.State.READY, - expiration_date=datetime.date(2023, 5, 25) + Domain.objects.get_or_create( + name="waterbutpurple.gov", state=Domain.State.READY, expiration_date=datetime.date(2023, 11, 15) + ) + Domain.objects.get_or_create( + name="fake.gov", state=Domain.State.READY, expiration_date=datetime.date(2022, 5, 25) + ) + Domain.objects.get_or_create( + name="fakeneeded.gov", state=Domain.State.DNS_NEEDED, expiration_date=datetime.date(2023, 11, 15) ) def tearDown(self): @@ -44,7 +48,7 @@ class TestExtendExpirationDates(MockEppLib): # Delete users User.objects.all().delete() UserDomainRole.objects.all().delete() - + def run_extend_expiration_dates(self): """ This method executes the transfer_transition_domains_to_domains command. @@ -57,47 +61,87 @@ class TestExtendExpirationDates(MockEppLib): return_value=True, ): call_command("extend_expiration_dates") - + def test_extends_expiration_date_correctly(self): - desired_domain = Domain.objects.filter(name="fake.gov").get() + """ + Tests that the extend_expiration_dates method extends dates as expected + """ + desired_domain = Domain.objects.filter(name="waterbutpurple.gov").get() desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) # Run the expiration date script self.run_extend_expiration_dates() - - self.assertEqual(desired_domain, self.domain) - # Explicitly test the expiration date - self.assertEqual(self.domain.expiration_date, datetime.date(2024, 5, 25)) - - # TODO ALSO NEED A TEST FOR NON READY DOMAINS + current_domain = Domain.objects.filter(name="waterbutpurple.gov").get() + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date + self.assertEqual(current_domain.expiration_date, datetime.date(2025, 1, 10)) + def test_extends_expiration_date_skips_non_current(self): + """ + Tests that the extend_expiration_dates method correctly skips domains + with an expiration date less than a certain threshold. + """ desired_domain = Domain.objects.filter(name="fake.gov").get() desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) # Run the expiration date script self.run_extend_expiration_dates() - - current_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + + current_domain = Domain.objects.filter(name="fake.gov").get() self.assertEqual(desired_domain, current_domain) # Explicitly test the expiration date. The extend_expiration_dates script # will skip all dates less than date(2023, 11, 15), meaning that this domain # should not be affected by the change. - self.assertEqual(current_domain.expiration_date, datetime.date(2023, 5, 25)) - - def test_extends_expiration_date_idempotent(self): - desired_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + self.assertEqual(current_domain.expiration_date, datetime.date(2022, 5, 25)) + + def test_extends_expiration_date_skips_non_ready(self): + """ + Tests that the extend_expiration_dates method correctly skips domains not in the state "ready" + """ + desired_domain = Domain.objects.filter(name="fakeneeded.gov").get() desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) # Run the expiration date script self.run_extend_expiration_dates() - - current_domain = Domain.objects.filter(name="FakeWebsite3.gov").get() + + current_domain = Domain.objects.filter(name="fake.gov").get() self.assertEqual(desired_domain, current_domain) - # Explicitly test the expiration date - self.assertEqual(desired_domain.expiration_date, datetime.date(2023, 9, 30)) + # Explicitly test the expiration date. The extend_expiration_dates script + # will skip all dates less than date(2023, 11, 15), meaning that this domain + # should not be affected by the change. + self.assertEqual(current_domain.expiration_date, datetime.date(2023, 11, 15)) + + def test_extends_expiration_date_idempotent(self): + """ + Tests the idempotency of the extend_expiration_dates command. + + Verifies that running the method multiple times does not change the expiration date + of a domain beyond the initial extension. + """ + desired_domain = Domain.objects.filter(name="waterbutpurple.gov").get() + desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + + # Run the expiration date script + self.run_extend_expiration_dates() + + current_domain = Domain.objects.filter(name="waterbutpurple.gov").get() + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date + self.assertEqual(desired_domain.expiration_date, datetime.date(2024, 11, 15)) + + # Run the expiration date script again + self.run_extend_expiration_dates() + + # The old domain shouldn't have changed + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date - should be the same + self.assertEqual(desired_domain.expiration_date, datetime.date(2024, 11, 15)) class TestOrganizationMigration(TestCase): From 2ce4a36a186467a33d45183aa3c424c5da4c54ac Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:34:27 -0700 Subject: [PATCH 07/78] Linting + test case typo --- .../tests/test_transition_domain_migrations.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4fbbcbfc8..4efdee43e 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -19,19 +19,21 @@ from unittest.mock import patch from registrar.models.contact import Contact from .common import MockEppLib, less_console_noise -from dateutil.relativedelta import relativedelta class TestExtendExpirationDates(MockEppLib): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" super().setUp() + # Create a valid domain that is updatable Domain.objects.get_or_create( name="waterbutpurple.gov", state=Domain.State.READY, expiration_date=datetime.date(2023, 11, 15) ) + # Create a domain with an invalid expiration date Domain.objects.get_or_create( name="fake.gov", state=Domain.State.READY, expiration_date=datetime.date(2022, 5, 25) ) + # Create a domain with an invalid state Domain.objects.get_or_create( name="fakeneeded.gov", state=Domain.State.DNS_NEEDED, expiration_date=datetime.date(2023, 11, 15) ) @@ -67,7 +69,7 @@ class TestExtendExpirationDates(MockEppLib): Tests that the extend_expiration_dates method extends dates as expected """ desired_domain = Domain.objects.filter(name="waterbutpurple.gov").get() - desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + desired_domain.expiration_date = datetime.date(2025, 1, 10) # Run the expiration date script self.run_extend_expiration_dates() @@ -84,7 +86,7 @@ class TestExtendExpirationDates(MockEppLib): with an expiration date less than a certain threshold. """ desired_domain = Domain.objects.filter(name="fake.gov").get() - desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + desired_domain.expiration_date = datetime.date(2022, 5, 25) # Run the expiration date script self.run_extend_expiration_dates() @@ -102,12 +104,12 @@ class TestExtendExpirationDates(MockEppLib): Tests that the extend_expiration_dates method correctly skips domains not in the state "ready" """ desired_domain = Domain.objects.filter(name="fakeneeded.gov").get() - desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + desired_domain.expiration_date = datetime.date(2023, 11, 15) # Run the expiration date script self.run_extend_expiration_dates() - current_domain = Domain.objects.filter(name="fake.gov").get() + current_domain = Domain.objects.filter(name="fakeneeded.gov").get() self.assertEqual(desired_domain, current_domain) # Explicitly test the expiration date. The extend_expiration_dates script @@ -123,7 +125,7 @@ class TestExtendExpirationDates(MockEppLib): of a domain beyond the initial extension. """ desired_domain = Domain.objects.filter(name="waterbutpurple.gov").get() - desired_domain.expiration_date = desired_domain.expiration_date + relativedelta(years=1) + desired_domain.expiration_date = datetime.date(2024, 11, 15) # Run the expiration date script self.run_extend_expiration_dates() From de91bbeda0d526b16fa4b66686aa36195ec4d8ba Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:10:05 -0700 Subject: [PATCH 08/78] Change import order --- src/registrar/management/commands/extend_expiration_dates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index fe25da504..574455786 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -8,13 +8,14 @@ from django.core.management import BaseCommand from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper -from dateutil.relativedelta import relativedelta try: from epplib.exceptions import TransportError except ImportError: pass +from dateutil.relativedelta import relativedelta + logger = logging.getLogger(__name__) From 40d75887319d4ba3ed781876d27552d359a4ea78 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:34:12 -0700 Subject: [PATCH 09/78] Change from relative delta to function --- .../commands/extend_expiration_dates.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 574455786..5f7980f63 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -123,14 +123,15 @@ class Command(BaseCommand): # == Helper functions == # def idempotence_check(self, domain, extension_amount): """Determines if the proposed operation violates idempotency""" - proposed_date = domain.expiration_date + relativedelta(years=extension_amount) + proposed_date = self.add_years(domain.expiration_date, extension_amount) # Because our migration data had a hard stop date, we can determine if our change # is valid simply checking if adding a year to our current date yields a greater date # than the proposed. # CAVEAT: This check stops working after a year has elapsed between when this script # was ran, and when it was ran again. This is good enough for now, but a more robust # solution would be a DB flag. - is_idempotent = proposed_date < (date.today() + relativedelta(years=extension_amount + 1)) + extension_from_today = self.add_years(date.today(), extension_amount + 1) + is_idempotent = proposed_date < extension_from_today return is_idempotent def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): @@ -234,3 +235,18 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ ) + + # We use this manual approach rather than relative delta due to our + # github localenv not having the package installed. + # Credit: https://stackoverflow.com/questions/15741618/add-one-year-in-current-date-python + def add_years(self, old_date, years): + """Return a date that's `years` years after the date (or datetime) + object `old_date`. Return the same calendar date (month and day) in the + destination year, if it exists, otherwise use the following day + (thus changing February 29 to March 1). + + """ + try: + return old_date.replace(year = old_date.year + years) + except ValueError: + return old_date + (date(old_date.year + years, 1, 1) - date(old_date.year, 1, 1)) From b4658e6da338e071c48bd27bcef6267d7801599a Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:20:45 -0700 Subject: [PATCH 10/78] Update extend_expiration_dates.py --- src/registrar/management/commands/extend_expiration_dates.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 5f7980f63..62e33fabc 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -14,7 +14,6 @@ try: except ImportError: pass -from dateutil.relativedelta import relativedelta logger = logging.getLogger(__name__) @@ -237,7 +236,7 @@ class Command(BaseCommand): ) # We use this manual approach rather than relative delta due to our - # github localenv not having the package installed. + # github localenv not having the package installed. # Credit: https://stackoverflow.com/questions/15741618/add-one-year-in-current-date-python def add_years(self, old_date, years): """Return a date that's `years` years after the date (or datetime) @@ -247,6 +246,6 @@ class Command(BaseCommand): """ try: - return old_date.replace(year = old_date.year + years) + return old_date.replace(year=old_date.year + years) except ValueError: return old_date + (date(old_date.year + years, 1, 1) - date(old_date.year, 1, 1)) From 018eb4f1f6c196868820aec92478f85ceeeab6f9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:54:42 -0700 Subject: [PATCH 11/78] Fix idempotent bug, minor code refactor --- .../commands/extend_expiration_dates.py | 76 ++++++------------- 1 file changed, 24 insertions(+), 52 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 62e33fabc..d6fc08b07 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -51,17 +51,10 @@ class Command(BaseCommand): """ Extends the expiration dates for valid domains. - It first retrieves the command line options and checks if the parse limit is a positive integer. - Then, it fetches the valid domains from the database and calculates the number of domains to change. If a parse limit is set and it's less than the total number of valid domains, the number of domains to change is set to the parse limit. - For each domain, it checks if the operation is idempotent. - If the idempotence check is not disabled and the operation is not idempotent, the domain is skipped. - Otherwise, the expiration date of the domain is extended. - - Finally, it logs a summary of the script run, - including the number of successful, failed, and skipped updates. + Includes an idempotence check. """ # Retrieve command line options @@ -79,17 +72,15 @@ class Command(BaseCommand): ).order_by("name") domains_to_change_count = valid_domains.count() - if limit_parse != 0 and limit_parse < domains_to_change_count: + if limit_parse != 0: domains_to_change_count = limit_parse + valid_domains = valid_domains[:limit_parse] # Determines if we should continue code execution or not. # If the user prompts 'N', a sys.exit() will be called. self.prompt_user_to_proceed(extension_amount, domains_to_change_count) - for i, domain in enumerate(valid_domains): - if limit_parse != 0 and i > limit_parse: - break - + for domain in valid_domains: is_idempotent = self.idempotence_check(domain, extension_amount) if not disable_idempotence and not is_idempotent: self.update_skipped.append(domain.name) @@ -124,12 +115,12 @@ class Command(BaseCommand): """Determines if the proposed operation violates idempotency""" proposed_date = self.add_years(domain.expiration_date, extension_amount) # Because our migration data had a hard stop date, we can determine if our change - # is valid simply checking if adding a year to our current date yields a greater date + # is valid simply checking if adding two years to our current date yields a greater date # than the proposed. # CAVEAT: This check stops working after a year has elapsed between when this script # was ran, and when it was ran again. This is good enough for now, but a more robust # solution would be a DB flag. - extension_from_today = self.add_years(date.today(), extension_amount + 1) + extension_from_today = self.add_years(date.today(), extension_amount + 2) is_idempotent = proposed_date < extension_from_today return is_idempotent @@ -144,7 +135,7 @@ class Command(BaseCommand): ==Proposed Changes== Domains to change: {domains_to_change_count} """, - prompt_title="Do you wish to modify Expiration Dates for the given Domains?", + prompt_title="Do you wish to proceed?", ) logger.info(f"{TerminalColors.MAGENTA}" "Preparing to extend expiration dates..." f"{TerminalColors.ENDC}") @@ -167,6 +158,23 @@ class Command(BaseCommand): update_success_count = len(self.update_success) update_failed_count = len(self.update_failed) update_skipped_count = len(self.update_skipped) + + # Prepare debug messages + debug_messages = { + "success": f"{TerminalColors.OKCYAN}Updated the following Domains: {self.update_success}{TerminalColors.ENDC}\n", + "skipped": f"{TerminalColors.YELLOW}Skipped the following Domains: {self.update_skipped}{TerminalColors.ENDC}\n", + "failed": f"{TerminalColors.FAIL}Failed to update the following Domains: {self.update_failed}{TerminalColors.ENDC}\n", + } + + # Print out a list of everything that was changed, if we have any changes to log. + # Otherwise, don't print anything. + TerminalHelper.print_conditional( + debug, + f"{debug_messages.get('success') if update_success_count > 0 else ''}" + f"{debug_messages.get('skipped') if update_skipped_count > 0 else ''}" + f"{debug_messages.get('failed') if update_failed_count > 0 else ''}", + ) + if update_failed_count == 0 and update_skipped_count == 0: logger.info( f"""{TerminalColors.OKGREEN} @@ -175,27 +183,7 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ ) - TerminalHelper.print_conditional( - debug, - f""" - {TerminalColors.OKGREEN} - Updated the following Domains: {self.update_success} - {TerminalColors.ENDC} - """, - ) elif update_failed_count == 0: - TerminalHelper.print_conditional( - debug, - f""" - {TerminalColors.OKGREEN} - Updated the following Domains: {self.update_success} - {TerminalColors.ENDC} - - {TerminalColors.YELLOW} - Skipped the following Domains: {self.update_skipped} - {TerminalColors.ENDC} - """, - ) logger.info( f"""{TerminalColors.YELLOW} ============= FINISHED =============== @@ -207,22 +195,6 @@ class Command(BaseCommand): """ ) else: - TerminalHelper.print_conditional( - debug, - f""" - {TerminalColors.OKGREEN} - Updated the following Domains: {self.update_success} - {TerminalColors.ENDC} - - {TerminalColors.YELLOW} - Skipped the following Domains: {self.update_skipped} - {TerminalColors.ENDC} - - {TerminalColors.FAIL} - Failed to update the following Domains: {self.update_failed} - {TerminalColors.ENDC} - """, - ) logger.info( f"""{TerminalColors.FAIL} ============= FINISHED =============== From ed902ccfe31fea9b1a6949f6ad3c565dbed294d4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:40:43 -0700 Subject: [PATCH 12/78] Fix expiration date bug --- .../management/commands/extend_expiration_dates.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index d6fc08b07..7e5b81d19 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -84,6 +84,9 @@ class Command(BaseCommand): is_idempotent = self.idempotence_check(domain, extension_amount) if not disable_idempotence and not is_idempotent: self.update_skipped.append(domain.name) + logger.info( + f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}" + ) else: self.extend_expiration_date_on_domain(domain, extension_amount, debug) @@ -113,14 +116,14 @@ class Command(BaseCommand): # == Helper functions == # def idempotence_check(self, domain, extension_amount): """Determines if the proposed operation violates idempotency""" - proposed_date = self.add_years(domain.expiration_date, extension_amount) + proposed_date = self.add_years(domain.registry_expiration_date, extension_amount) # Because our migration data had a hard stop date, we can determine if our change # is valid simply checking if adding two years to our current date yields a greater date # than the proposed. # CAVEAT: This check stops working after a year has elapsed between when this script # was ran, and when it was ran again. This is good enough for now, but a more robust # solution would be a DB flag. - extension_from_today = self.add_years(date.today(), extension_amount + 2) + extension_from_today = self.add_years(date.today(), extension_amount + 1) is_idempotent = proposed_date < extension_from_today return is_idempotent From 9f9646f0959060e6d043d5e8cb32527d2b6e7f08 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:22:54 -0700 Subject: [PATCH 13/78] Linting change --- .../commands/extend_expiration_dates.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 7e5b81d19..42f26377e 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -84,9 +84,7 @@ class Command(BaseCommand): is_idempotent = self.idempotence_check(domain, extension_amount) if not disable_idempotence and not is_idempotent: self.update_skipped.append(domain.name) - logger.info( - f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") else: self.extend_expiration_date_on_domain(domain, extension_amount, debug) @@ -164,9 +162,15 @@ class Command(BaseCommand): # Prepare debug messages debug_messages = { - "success": f"{TerminalColors.OKCYAN}Updated the following Domains: {self.update_success}{TerminalColors.ENDC}\n", - "skipped": f"{TerminalColors.YELLOW}Skipped the following Domains: {self.update_skipped}{TerminalColors.ENDC}\n", - "failed": f"{TerminalColors.FAIL}Failed to update the following Domains: {self.update_failed}{TerminalColors.ENDC}\n", + "success": ( + f"{TerminalColors.OKCYAN}Updated these Domains: {self.update_success}{TerminalColors.ENDC}\n" + ), + "skipped": ( + f"{TerminalColors.YELLOW}Skipped these Domains: {self.update_skipped}{TerminalColors.ENDC}\n" + ), + "failed": ( + f"{TerminalColors.FAIL}Failed to update these Domains: {self.update_failed}{TerminalColors.ENDC}\n" + ), } # Print out a list of everything that was changed, if we have any changes to log. From d1b24ab6a294fc441c3116fa8b605eab25aa9e37 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:48:11 -0700 Subject: [PATCH 14/78] Add PR changes --- src/registrar/admin.py | 1 + .../commands/load_transition_domain.py | 28 ++-- .../transfer_transition_domains_to_domains.py | 16 +- .../utility/extra_transition_domain_helper.py | 12 +- src/registrar/models/transition_domain.py | 6 + .../test_transition_domain_migrations.py | 149 ++++++++++++++++++ 6 files changed, 195 insertions(+), 17 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 429bd762f..3b8f7c962 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -752,6 +752,7 @@ class TransitionDomainAdmin(ListHeaderAdmin): "domain_name", "status", "email_sent", + "processed", ] search_fields = ["username", "domain_name"] diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index e1165bf9f..4132096c8 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -536,19 +536,27 @@ class Command(BaseCommand): domain_name=new_entry_domain_name, ) - if existing_entry.status != new_entry_status: - # DEBUG: + if not existing_entry.processed: + if existing_entry.status != new_entry_status: + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN}" + f"Updating entry: {existing_entry}" + f"Status: {existing_entry.status} > {new_entry_status}" # noqa + f"Email Sent: {existing_entry.email_sent} > {new_entry_emailSent}" # noqa + f"{TerminalColors.ENDC}", + ) + existing_entry.status = new_entry_status + existing_entry.email_sent = new_entry_emailSent + existing_entry.save() + else: TerminalHelper.print_conditional( debug_on, - f"{TerminalColors.OKCYAN}" - f"Updating entry: {existing_entry}" - f"Status: {existing_entry.status} > {new_entry_status}" # noqa - f"Email Sent: {existing_entry.email_sent} > {new_entry_emailSent}" # noqa + f"{TerminalColors.YELLOW}" + f"Skipping update on processed domain: {existing_entry}" f"{TerminalColors.ENDC}", ) - existing_entry.status = new_entry_status - existing_entry.email_sent = new_entry_emailSent - existing_entry.save() + except TransitionDomain.MultipleObjectsReturned: logger.info( f"{TerminalColors.FAIL}" @@ -558,6 +566,7 @@ class Command(BaseCommand): f"----------TERMINATING----------" ) sys.exit() + else: # no matching entry, make one new_entry = TransitionDomain( @@ -565,6 +574,7 @@ class Command(BaseCommand): domain_name=new_entry_domain_name, status=new_entry_status, email_sent=new_entry_emailSent, + processed=False, ) to_create.append(new_entry) total_new_entries += 1 diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index d0d6ff363..15cd7376d 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -559,7 +559,8 @@ class Command(BaseCommand): debug_max_entries_to_parse, total_rows_parsed, ): - for transition_domain in TransitionDomain.objects.all(): + changed_transition_domains = TransitionDomain.objects.filter(processed=False) + for transition_domain in changed_transition_domains: ( target_domain_information, associated_domain, @@ -644,7 +645,8 @@ class Command(BaseCommand): debug_max_entries_to_parse, total_rows_parsed, ): - for transition_domain in TransitionDomain.objects.all(): + changed_transition_domains = TransitionDomain.objects.filter(processed=False) + for transition_domain in changed_transition_domains: # Create some local variables to make data tracing easier transition_domain_name = transition_domain.domain_name transition_domain_status = transition_domain.status @@ -796,6 +798,7 @@ class Command(BaseCommand): # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) + # DomainInvitation.objects.bulk_create(domain_invitations_to_create) # TODO: this is to resolve an error where bulk_create @@ -847,6 +850,15 @@ class Command(BaseCommand): ) DomainInformation.objects.bulk_create(domain_information_to_create) + # Loop through the list of everything created, and mark it as processed + for domain in domains_to_create: + name = domain.name + TransitionDomain.objects.filter(domain_name=name).update(processed=True) + + # Loop through the list of everything updated, and mark it as processed + for name in updated_domain_entries: + TransitionDomain.objects.filter(domain_name=name).update(processed=True) + self.print_summary_of_findings( domains_to_create, updated_domain_entries, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 04170811f..54f68d5c8 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -155,13 +155,13 @@ class LoadExtraTransitionDomain: def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.parsed_data_container""" - all_transition_domains = TransitionDomain.objects.all() - if not all_transition_domains.exists(): - raise ValueError("No TransitionDomain objects exist.") + valid_transition_domains = TransitionDomain.objects.filter(processed=False) + if not valid_transition_domains.exists(): + raise ValueError("No updatable TransitionDomain objects exist.") updated_transition_domains = [] failed_transition_domains = [] - for transition_domain in all_transition_domains: + for transition_domain in valid_transition_domains: domain_name = transition_domain.domain_name updated_transition_domain = transition_domain try: @@ -228,7 +228,7 @@ class LoadExtraTransitionDomain: # DATA INTEGRITY CHECK # Make sure every Transition Domain got updated total_transition_domains = len(updated_transition_domains) - total_updates_made = TransitionDomain.objects.all().count() + total_updates_made = TransitionDomain.objects.filter(processed=False).count() if total_transition_domains != total_updates_made: # noqa here for line length logger.error( @@ -787,7 +787,7 @@ class OrganizationDataLoader: self.tds_to_update: List[TransitionDomain] = [] def update_organization_data_for_all(self): - """Updates org address data for all TransitionDomains""" + """Updates org address data for valid TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) == 0: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE) diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 28bdc4fc7..6fe230951 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -43,6 +43,12 @@ class TransitionDomain(TimeStampedModel): verbose_name="email sent", help_text="indicates whether email was sent", ) + processed = models.BooleanField( + null=False, + default=True, + verbose_name="Processed", + help_text="Indicates whether this TransitionDomain was already processed", + ) organization_type = models.TextField( max_length=255, null=True, diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4e549bdd6..cfee68fea 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -21,6 +21,155 @@ from registrar.models.contact import Contact from .common import less_console_noise +class TestProcessedMigrations(TestCase): + """This test case class is designed to verify the idempotency of migrations + related to domain transitions in the application.""" + + def setUp(self): + """Defines the file name of migration_json and the folder its contained in""" + self.test_data_file_location = "registrar/tests/data" + self.migration_json_filename = "test_migrationFilepaths.json" + self.user, _ = User.objects.get_or_create(username="igorvillian") + + def tearDown(self): + """Deletes all DB objects related to migrations""" + # Delete domain information + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainInvitation.objects.all().delete() + TransitionDomain.objects.all().delete() + + # Delete users + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def run_load_domains(self): + """ + This method executes the load_transition_domain command. + + It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, + which is a user prompt in the terminal. The mock function always returns True, + allowing the test to proceed without manual user input. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + # noqa here because splitting this up makes it confusing. + # ES501 + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_transition_domain", + self.migration_json_filename, + directory=self.test_data_file_location, + ) + + def run_transfer_domains(self): + """ + This method executes the transfer_transition_domains_to_domains command. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + call_command("transfer_transition_domains_to_domains") + + def test_domain_idempotent(self): + """ + This test ensures that the domain transfer process + is idempotent on Domain and DomainInformation. + """ + unchanged_domain, _ = Domain.objects.get_or_create( + name="testdomain.gov", + state=Domain.State.READY, + expiration_date=datetime.date(2000, 1, 1), + ) + unchanged_domain_information, _ = DomainInformation.objects.get_or_create( + domain=unchanged_domain, organization_name="test org name", creator=self.user + ) + self.run_load_domains() + + # Test that a given TransitionDomain isn't set to "processed" + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertFalse(transition_domain_object.processed) + + self.run_transfer_domains() + + # Test that old data isn't corrupted + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Test that a given TransitionDomain is set to "processed" after we transfer domains + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertTrue(transition_domain_object.processed) + + # Manually change Domain/DomainInformation objects + changed_domain = Domain.objects.filter(name="fakewebsite3.gov").get() + changed_domain.expiration_date = datetime.date(1999, 1, 1) + + changed_domain.save() + + changed_domain_information = DomainInformation.objects.filter(domain=changed_domain).get() + changed_domain_information.organization_name = "changed" + + changed_domain_information.save() + + # Rerun transfer domains + self.run_transfer_domains() + + # Test that old data isn't corrupted after running this twice + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Ensure that domain hasn't changed + actual_domain = Domain.objects.filter(name="fakewebsite3.gov").get() + self.assertEqual(changed_domain, actual_domain) + + # Ensure that DomainInformation hasn't changed + actual_domain_information = DomainInformation.objects.filter(domain=changed_domain).get() + self.assertEqual(changed_domain_information, actual_domain_information) + + def test_transition_domain_is_processed(self): + """ + This test checks if a domain is correctly marked as processed in the transition. + """ + old_transition_domain, _ = TransitionDomain.objects.get_or_create(domain_name="testdomain.gov") + # Asser that old records default to 'True' + self.assertTrue(old_transition_domain.processed) + + unchanged_domain, _ = Domain.objects.get_or_create( + name="testdomain.gov", + state=Domain.State.READY, + expiration_date=datetime.date(2000, 1, 1), + ) + unchanged_domain_information, _ = DomainInformation.objects.get_or_create( + domain=unchanged_domain, organization_name="test org name", creator=self.user + ) + self.run_load_domains() + + # Test that a given TransitionDomain isn't set to "processed" + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertFalse(transition_domain_object.processed) + + self.run_transfer_domains() + + # Test that old data isn't corrupted + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertTrue(old_transition_domain.processed) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Test that a given TransitionDomain is set to "processed" after we transfer domains + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertTrue(transition_domain_object.processed) + + class TestOrganizationMigration(TestCase): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" From 6970adb57fdf604b690982191e58fa890a1bbf35 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:04:58 -0700 Subject: [PATCH 15/78] Linter --- .../management/commands/extend_expiration_dates.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 42f26377e..2242bbeee 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -162,12 +162,8 @@ class Command(BaseCommand): # Prepare debug messages debug_messages = { - "success": ( - f"{TerminalColors.OKCYAN}Updated these Domains: {self.update_success}{TerminalColors.ENDC}\n" - ), - "skipped": ( - f"{TerminalColors.YELLOW}Skipped these Domains: {self.update_skipped}{TerminalColors.ENDC}\n" - ), + "success": (f"{TerminalColors.OKCYAN}Updated these Domains: {self.update_success}{TerminalColors.ENDC}\n"), + "skipped": (f"{TerminalColors.YELLOW}Skipped these Domains: {self.update_skipped}{TerminalColors.ENDC}\n"), "failed": ( f"{TerminalColors.FAIL}Failed to update these Domains: {self.update_failed}{TerminalColors.ENDC}\n" ), From e25dad495ef2122edd4703142e3ff94b8c42ebf5 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 11 Dec 2023 17:20:47 -0500 Subject: [PATCH 16/78] Intercept userinfo from login.gov and append given_name and family_name if user requires ial1 and exists in DB --- src/djangooidc/oidc.py | 17 +++++++++-------- src/djangooidc/views.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/djangooidc/oidc.py b/src/djangooidc/oidc.py index 91bfddc66..331490cae 100644 --- a/src/djangooidc/oidc.py +++ b/src/djangooidc/oidc.py @@ -87,7 +87,7 @@ class Client(oic.Client): extra_args=None, ): """Step 2: Construct a login URL at OP's domain and send the user to it.""" - logger.debug("Creating the OpenID Connect authn request...") + logger.info("create_authn_request() Creating the OpenID Connect authn request...") state = rndstr(size=32) try: session["state"] = state @@ -112,7 +112,7 @@ class Client(oic.Client): logger.error("Failed to assemble request arguments for %s" % state) raise o_e.InternalError(locator=state) - logger.debug("request args: %s" % request_args) + logger.info("request args: %s" % request_args) try: # prepare the request for sending @@ -126,9 +126,9 @@ class Client(oic.Client): method="GET", request_args=request_args, ) - logger.debug("body: %s" % body) - logger.debug("URL: %s" % url) - logger.debug("headers: %s" % headers) + logger.info("body: %s" % body) + logger.info("URL: %s" % url) + logger.info("headers: %s" % headers) except Exception as err: logger.error(err) logger.error("Failed to prepare request for %s" % state) @@ -150,7 +150,7 @@ class Client(oic.Client): def callback(self, unparsed_response, session): """Step 3: Receive OP's response, request an access token, and user info.""" - logger.debug("Processing the OpenID Connect callback response...") + logger.info("callback() Processing the OpenID Connect callback response...") state = session.get("state", "") try: # parse the response from OP @@ -174,7 +174,7 @@ class Client(oic.Client): logger.error("Unable to process response %s for %s" % (error, state)) raise o_e.AuthenticationFailed(locator=state) - logger.debug("authn_response %s" % authn_response) + logger.info("callback() authn_response %s" % authn_response) if not authn_response.get("state", None): logger.error("State value not received from OP for %s" % state) @@ -213,7 +213,8 @@ class Client(oic.Client): logger.error("Unable to get user info (%s) for %s" % (info_response.get("error", ""), state)) raise o_e.AuthenticationFailed(locator=state) - logger.debug("user info: %s" % info_response) + logger.info("_get_user_info() user info: %s" % info_response) + return info_response.to_dict() def _request_token(self, state, code, session): diff --git a/src/djangooidc/views.py b/src/djangooidc/views.py index f354a43b4..2ce26adaa 100644 --- a/src/djangooidc/views.py +++ b/src/djangooidc/views.py @@ -58,6 +58,7 @@ def openid(request): request.session["next"] = request.GET.get("next", "/") try: + logger.info('openid() calls create_authn_request in oidc') return CLIENT.create_authn_request(request.session) except Exception as err: return error_page(request, err) @@ -71,9 +72,24 @@ def login_callback(request): # test for need for identity verification and if it is satisfied # if not satisfied, redirect user to login with stepped up acr_value if requires_step_up_auth(userinfo): + logger.info('login_callback() calls get_step_up_acr_value and create_authn_request in oidc') # add acr_value to request.session request.session["acr_value"] = CLIENT.get_step_up_acr_value() return CLIENT.create_authn_request(request.session) + + logger.info(f'login_callback() before calling authenticate: {userinfo}') + + try: + user_in_db = User.objects.get(username=userinfo["sub"]) + + if user_in_db: + logger.info(f"This user exists in the DB (before authenticate): {user_in_db.first_name}") + userinfo["given_name"] = user_in_db.first_name + userinfo["family_name"] = user_in_db.last_name + except: + pass + + user = authenticate(request=request, **userinfo) if user: login(request, user) From 23db6dea9dd4da5aed72022c388dc9014bc15396 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:48:37 -0700 Subject: [PATCH 17/78] Fix _get_or_create_domain bug --- src/registrar/models/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 94430fb36..53ea302eb 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1194,7 +1194,7 @@ class Domain(TimeStampedModel, DomainHelper): logger.error(e) logger.error(e.code) raise e - if e.code == ErrorCode.OBJECT_DOES_NOT_EXIST and self.state != Domain.State.DELETED: + if e.code == ErrorCode.OBJECT_DOES_NOT_EXIST and self.state == Domain.State.UNKNOWN: # avoid infinite loop already_tried_to_create = True self.dns_needed_from_unknown() From fd19726a30edddc9cbc98a310f0211696f5a6938 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 11 Dec 2023 19:03:10 -0500 Subject: [PATCH 18/78] updated logic in oidc backend authenticate to not override with blank first_name/last_name --- src/djangooidc/backends.py | 15 +++++++++++++-- src/djangooidc/views.py | 4 +--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/djangooidc/backends.py b/src/djangooidc/backends.py index ce6b3acd3..d4df7aa50 100644 --- a/src/djangooidc/backends.py +++ b/src/djangooidc/backends.py @@ -46,8 +46,19 @@ class OpenIdConnectBackend(ModelBackend): # defaults _will_ be updated, these are not fallbacks "defaults": openid_data, } - user, created = UserModel.objects.update_or_create(**args) - if created: + + user, created = UserModel.objects.get_or_create(**args) + + if not created: + # User already exists, update other fields without overwriting first_name and last_name + # overwrite first_name and last_name if not empty string + for key, value in args["defaults"].items(): + # Check if the key is not first_name or last_name or value is not empty string + if key not in ['first_name', 'last_name'] or value != "": + setattr(user, key, value) + user.save() + else: + # If user is created, configure the user user = self.configure_user(user, **kwargs) else: try: diff --git a/src/djangooidc/views.py b/src/djangooidc/views.py index 3a5c5fd20..7efb576da 100644 --- a/src/djangooidc/views.py +++ b/src/djangooidc/views.py @@ -83,9 +83,7 @@ def login_callback(request): user_in_db = User.objects.get(username=userinfo["sub"]) if user_in_db: - logger.info(f"This user exists in the DB (before authenticate): {user_in_db.first_name}") - userinfo["given_name"] = user_in_db.first_name - userinfo["family_name"] = user_in_db.last_name + logger.info(f"This user exists in the DB (before authenticate): {user_in_db.first_name} {user_in_db.last_name}") except: pass From e4803d6afdbc21fba4fc3578f8a3a93e5b0ea48b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 11 Dec 2023 19:08:34 -0500 Subject: [PATCH 19/78] refactored authenticate for readability --- src/djangooidc/backends.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/djangooidc/backends.py b/src/djangooidc/backends.py index d4df7aa50..d77a3fe15 100644 --- a/src/djangooidc/backends.py +++ b/src/djangooidc/backends.py @@ -50,13 +50,8 @@ class OpenIdConnectBackend(ModelBackend): user, created = UserModel.objects.get_or_create(**args) if not created: - # User already exists, update other fields without overwriting first_name and last_name - # overwrite first_name and last_name if not empty string - for key, value in args["defaults"].items(): - # Check if the key is not first_name or last_name or value is not empty string - if key not in ['first_name', 'last_name'] or value != "": - setattr(user, key, value) - user.save() + # If user exists, update existing user + self.update_existing_user(user, args["defaults"]) else: # If user is created, configure the user user = self.configure_user(user, **kwargs) @@ -69,6 +64,15 @@ class OpenIdConnectBackend(ModelBackend): user.on_each_login() return user + def update_existing_user(self, user, kwargs): + # Update other fields without overwriting first_name and last_name. + # Overwrite first_name and last_name if not empty string + for key, value in kwargs.items(): + # Check if the key is not first_name or last_name or value is not empty string + if key not in ['first_name', 'last_name'] or value: + setattr(user, key, value) + user.save() + def clean_username(self, username): """ Performs any cleaning on the "username" prior to using it to get or From ae811d16a9b76c86c049f391cd6eb6d6e22066ef Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 11 Dec 2023 19:15:08 -0500 Subject: [PATCH 20/78] Contact saves update the corresponding user's first and last names --- src/registrar/models/contact.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index 0a7ba4fa1..6b3b6ddb2 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -59,6 +59,16 @@ class Contact(TimeStampedModel): names = [n for n in [self.first_name, self.middle_name, self.last_name] if n] return " ".join(names) if names else "Unknown" + def save(self, *args, **kwargs): + # Call the parent class's save method to perform the actual save + super().save(*args, **kwargs) + + # Update the related User object's first_name and last_name + if self.user: + self.user.first_name = self.first_name + self.user.last_name = self.last_name + self.user.save() + def __str__(self): if self.first_name or self.last_name: return self.get_formatted_name() From e7c26d9dc636effe81589919daa9b7c727649560 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 11 Dec 2023 20:03:00 -0500 Subject: [PATCH 21/78] wrote test cases for testing backends.py authenticate --- src/djangooidc/tests/test_backends.py | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/djangooidc/tests/test_backends.py diff --git a/src/djangooidc/tests/test_backends.py b/src/djangooidc/tests/test_backends.py new file mode 100644 index 000000000..3283e2cc4 --- /dev/null +++ b/src/djangooidc/tests/test_backends.py @@ -0,0 +1,82 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from django.utils import timezone +from registrar.models import User +from ..backends import OpenIdConnectBackend # Adjust the import path based on your project structure + +class OpenIdConnectBackendTestCase(TestCase): + + def setUp(self): + self.backend = OpenIdConnectBackend() + self.kwargs = { + "sub": "test_user", + "given_name": "John", + "family_name": "Doe", + "email": "john.doe@example.com", + "phone": "123456789", + } + + def tearDown(self) -> None: + User.objects.all().delete() + + def test_authenticate_with_create_user(self): + """Test that authenticate creates a new user if it does not find + existing user""" + # Ensure that the authenticate method creates a new user + user = self.backend.authenticate(request=None, **self.kwargs) + self.assertIsNotNone(user) + self.assertIsInstance(user, User) + self.assertEqual(user.username, "test_user") + + # Verify that user fields are correctly set + self.assertEqual(user.first_name, "John") + self.assertEqual(user.last_name, "Doe") + self.assertEqual(user.email, "john.doe@example.com") + self.assertEqual(user.phone, "123456789") + + def test_authenticate_with_existing_user(self): + """Test that authenticate updates an existing user if it finds one. + For this test, given_name and family_name are supplied""" + # Create an existing user with the same username + existing_user = User.objects.create_user(username="test_user") + + # Ensure that the authenticate method updates the existing user + user = self.backend.authenticate(request=None, **self.kwargs) + self.assertIsNotNone(user) + self.assertIsInstance(user, User) + self.assertEqual(user, existing_user) # The same user instance should be returned + + # Verify that user fields are correctly updated + self.assertEqual(user.first_name, "John") + self.assertEqual(user.last_name, "Doe") + self.assertEqual(user.email, "john.doe@example.com") + self.assertEqual(user.phone, "123456789") + + def test_authenticate_with_existing_user_no_name(self): + """Test that authenticate updates an existing user if it finds one. + For this test, given_name and family_name are supplied""" + # Create an existing user with the same username and with first and last names + existing_user = User.objects.create_user(username="test_user",first_name="John",last_name="Doe") + + # Remove given_name and family_name from the input, self.kwargs + self.kwargs.pop("given_name", None) + self.kwargs.pop("family_name", None) + + # Ensure that the authenticate method updates the existing user + # and preserves existing first and last names + user = self.backend.authenticate(request=None, **self.kwargs) + self.assertIsNotNone(user) + self.assertIsInstance(user, User) + self.assertEqual(user, existing_user) # The same user instance should be returned + + # Verify that user fields are correctly updated + self.assertEqual(user.first_name, "John") + self.assertEqual(user.last_name, "Doe") + self.assertEqual(user.email, "john.doe@example.com") + self.assertEqual(user.phone, "123456789") + + def test_authenticate_with_unknown_user(self): + """Test that authenticate returns None when no kwargs are supplied""" + # Ensure that the authenticate method handles the case when the user is not found + user = self.backend.authenticate(request=None, **{}) + self.assertIsNone(user) From abae70b96e8477f1e01181f9189c49f4abd48bf9 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 11 Dec 2023 20:49:33 -0500 Subject: [PATCH 22/78] added unit test for conflicting first and last names --- src/djangooidc/tests/test_backends.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/djangooidc/tests/test_backends.py b/src/djangooidc/tests/test_backends.py index 3283e2cc4..51fc965c4 100644 --- a/src/djangooidc/tests/test_backends.py +++ b/src/djangooidc/tests/test_backends.py @@ -54,7 +54,7 @@ class OpenIdConnectBackendTestCase(TestCase): def test_authenticate_with_existing_user_no_name(self): """Test that authenticate updates an existing user if it finds one. - For this test, given_name and family_name are supplied""" + For this test, given_name and family_name are not supplied""" # Create an existing user with the same username and with first and last names existing_user = User.objects.create_user(username="test_user",first_name="John",last_name="Doe") @@ -75,6 +75,25 @@ class OpenIdConnectBackendTestCase(TestCase): self.assertEqual(user.email, "john.doe@example.com") self.assertEqual(user.phone, "123456789") + def test_authenticate_with_existing_user_different_name(self): + """Test that authenticate updates an existing user if it finds one. + For this test, given_name and family_name are supplied and overwrite""" + # Create an existing user with the same username and with first and last names + existing_user = User.objects.create_user(username="test_user",first_name="WillBe",last_name="Replaced") + + # Ensure that the authenticate method updates the existing user + # and preserves existing first and last names + user = self.backend.authenticate(request=None, **self.kwargs) + self.assertIsNotNone(user) + self.assertIsInstance(user, User) + self.assertEqual(user, existing_user) # The same user instance should be returned + + # Verify that user fields are correctly updated + self.assertEqual(user.first_name, "John") + self.assertEqual(user.last_name, "Doe") + self.assertEqual(user.email, "john.doe@example.com") + self.assertEqual(user.phone, "123456789") + def test_authenticate_with_unknown_user(self): """Test that authenticate returns None when no kwargs are supplied""" # Ensure that the authenticate method handles the case when the user is not found From 48f69c89667f65be0d5a2b11a3693409159f3833 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 11 Dec 2023 21:29:38 -0500 Subject: [PATCH 23/78] Unit tests to asert that updating a contact's first and last names (but not an email) propagate to the linked user --- src/registrar/tests/test_models.py | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index ba58ad858..c51af8008 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -654,3 +654,50 @@ class TestUser(TestCase): """A new user who's neither transitioned nor invited should return True when tested with class method needs_identity_verification""" self.assertTrue(User.needs_identity_verification(self.user.email, self.user.username)) + + +class TestContact(TestCase): + + def setUp(self): + self.email = "mayor@igorville.gov" + self.user, _ = User.objects.get_or_create(email=self.email, first_name="Rachid", last_name="Mrad") + self.contact, _ = Contact.objects.get_or_create(user=self.user) + + def tearDown(self): + super().tearDown() + Contact.objects.all().delete() + User.objects.all().delete() + + def test_saving_contact_updates_user_first_last_names(self): + """When a contact is updated, we propagate the changes to the linked user if it exists.""" + # User and Contact are created and linked as expected + self.assertEqual(self.contact.first_name, "Rachid") + self.assertEqual(self.contact.last_name, "Mrad") + self.assertEqual(self.user.first_name, "Rachid") + self.assertEqual(self.user.last_name, "Mrad") + + self.contact.first_name = "Joey" + self.contact.last_name = "Baloney" + self.contact.save() + + # Refresh the user object to reflect the changes made in the database + self.user.refresh_from_db() + + # Updating the contact's first and last names propagate to the user + self.assertEqual(self.contact.first_name, "Joey") + self.assertEqual(self.contact.last_name, "Baloney") + self.assertEqual(self.user.first_name, "Joey") + self.assertEqual(self.user.last_name, "Baloney") + + def test_saving_contact_does_not_update_user_email(self): + """When a contact's email is updated, the change is not propagated to the lined user.""" + self.contact.email = "joey.baloney@diaperville.com" + self.contact.save() + + # Refresh the user object to reflect the changes made in the database + self.user.refresh_from_db() + + # Updating the contact's email does not propagate + self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") + self.assertEqual(self.user.email, "mayor@igorville.gov") + From b7da6e58f3753799e33f3f2412c0bd74ed1e2d44 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 11 Dec 2023 23:25:42 -0500 Subject: [PATCH 24/78] WIP script to copy first and last from contacts into users --- .../copy_names_from_contacts_to_users.py | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 src/registrar/management/commands/copy_names_from_contacts_to_users.py diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py new file mode 100644 index 000000000..779788df9 --- /dev/null +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -0,0 +1,269 @@ +import logging +import argparse +import sys + +from django.core.management import BaseCommand + +from registrar.management.commands.utility.terminal_helper import ( + TerminalColors, + TerminalHelper, +) +from registrar.models.contact import Contact +from registrar.models.user import User + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = """Copy first and last names from a contact to + a related user if it exists and if its first and last name + properties are null""" + + # ====================================================== + # ===================== ARGUMENTS ===================== + # ====================================================== + def add_arguments(self, parser): + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument( + "--limitParse", + default=0, + help="Sets max number of records (contacts) to copy, set to 0 to copy all entries", + ) + + # ====================================================== + # ===================== PRINTING ====================== + # ====================================================== + def print_debug_mode_statements(self, debug_on: bool, debug_max_entries_to_parse: int): + """Prints additional terminal statements to indicate if --debug + or --limitParse are in use""" + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.OKCYAN} + ----------DEBUG MODE ON---------- + Detailed print statements activated. + {TerminalColors.ENDC} + """, + ) + TerminalHelper.print_conditional( + debug_max_entries_to_parse > 0, + f"""{TerminalColors.OKCYAN} + ----------LIMITER ON---------- + Parsing of entries will be limited to + {debug_max_entries_to_parse} lines per file.") + Detailed print statements activated. + {TerminalColors.ENDC} + """, + ) + + def parse_limit_reached(self, debug_max_entries_to_parse: bool, total_rows_parsed: int) -> bool: + if debug_max_entries_to_parse > 0 and total_rows_parsed/2 >= debug_max_entries_to_parse: + logger.info( + f"""{TerminalColors.YELLOW} + ----PARSE LIMIT REACHED. HALTING PARSER.---- + {TerminalColors.ENDC} + """ + ) + return True + return False + + def print_summary_of_findings( + self, + updated_user_records, + skipped_contacts, + debug_on, + ): + """Prints to terminal a summary of findings from + copying first and last names from contacts to users""" + + total_updated_user_entries = len(updated_user_records) + total_skipped_contacts_entries = len(skipped_contacts) + + logger.info( + f"""{TerminalColors.OKGREEN} + ============= FINISHED =============== + Updated {total_updated_user_entries} users + Skipped {total_skipped_contacts_entries} contacts + {TerminalColors.ENDC} + """ # noqa + ) + + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + ======= DEBUG OUTPUT ======= + Updated User records: + {updated_user_records} + + ===== SKIPPED CONTACTS ===== + {skipped_contacts} + + {TerminalColors.ENDC} + """, + ) + + # ====================================================== + # =================== USER ===================== + # ====================================================== + def update_user(self, contact: Contact, debug_on: bool): + """Given a contact with a first_name and last_name, find & update an existing + corresponding user if her first_name and last_name are null. + + Returns the corresponding User object. + """ + + # Create some local variables to make data tracing easier + contact_email = contact.email + contact_first_name = contact.first_name + contact_lastname = contact.last_name + + user_exists = User.objects.filter(contact=contact).exists() + if user_exists: + try: + # ----------------------- UPDATE USER ----------------------- + # ---- GET THE USER + target_user = User.objects.get(contact=contact) + # DEBUG: + TerminalHelper.print_conditional( + debug_on, + f"""{TerminalColors.YELLOW} + > Found linked entry in User table for: {contact_first_name} {contact_lastname} {contact_email} + {TerminalColors.ENDC}""", # noqa + ) + + # ---- UPDATE THE USER IF IT DOES NOT HAVE A FIRST AND LAST NAMES + # ---- LET'S KEEP A LIGHT TOUCH + if not target_user.first_name or not target_user.last_name: + target_user.first_name = contact_first_name + target_user.last_name = contact_lastname + + target_user.save() + + return (target_user) + + except Exception as E: + logger.warning( + f""" + {TerminalColors.FAIL} + !!! ERROR: An exception occured in the + User table for the following user: + {contact_email} + ----------TERMINATING----------""" + ) + sys.exit() + + # ====================================================== + # ================= PROCESS CONTACTS ================== + # ====================================================== + + # C901 'Command.handle' is too complex + def process_contacts( + self, + debug_on, + skipped_user_entries, + updated_user_entries, + debug_max_entries_to_parse, + total_rows_parsed, + ): + for contact in Contact.objects.all(): + # Create some local variables to make data tracing easier + contact_email = contact.email + contact_first_name = contact.first_name + contact_last_name = contact.last_name + + # DEBUG: + # TerminalHelper.print_conditional( + # debug_on, + # f"{TerminalColors.OKCYAN}" + # "Processing Contact: " + # f"{contact_email}," + # f" {contact_first_name}," + # f" {contact_last_name}" + # f"{TerminalColors.ENDC}", # noqa + # ) + + # ====================================================== + # ====================== USER ======================= + target_user = self.update_user(contact, debug_on) + + debug_string = "" + if target_user: + # ---------------- UPDATED ---------------- + updated_user_entries.append(contact.email) + debug_string = f"updated user: {target_user}" + else: + skipped_user_entries.append(contact.email) + debug_string = f"skipped user: {contact.email}" + + # DEBUG: + # TerminalHelper.print_conditional( + # debug_on, + # (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), + # ) + + # ------------------ Parse limit reached? ------------------ + # Check parse limit and exit loop if parse limit has been reached + if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): + break + return ( + skipped_user_entries, + updated_user_entries, + ) + + # ====================================================== + # ===================== HANDLE ======================== + # ====================================================== + def handle( + self, + **options, + ): + """Parse entries in Contact table + and update valid corresponding entries in the + User table.""" + + # grab command line arguments and store locally... + debug_on = options.get("debug") + debug_max_entries_to_parse = int(options.get("limitParse")) # set to 0 to parse all entries + + self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) + + # users we UPDATED + updated_user_entries = [] + + # users we SKIPPED + skipped_user_entries = [] + + # if we are limiting our parse (for testing purposes, keep + # track of total rows parsed) + total_rows_parsed = 0 + + logger.info( + f"""{TerminalColors.OKCYAN} + ========================== + Beginning Data Transfer + ========================== + {TerminalColors.ENDC}""" + ) + + logger.info( + f"""{TerminalColors.OKCYAN} + ========= Adding Domains and Domain Invitations ========= + {TerminalColors.ENDC}""" + ) + ( + skipped_user_entries, + updated_user_entries, + ) = self.process_contacts( + debug_on, + skipped_user_entries, + updated_user_entries, + debug_max_entries_to_parse, + total_rows_parsed, + ) + + self.print_summary_of_findings( + updated_user_entries, + skipped_user_entries, + debug_on, + ) From 89a04edfe6af5f2bf35c5a663bd2f91a4225fe9f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 08:55:29 -0700 Subject: [PATCH 25/78] Update extend_expiration_dates.py --- .../commands/extend_expiration_dates.py | 77 +++++++++---------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 2242bbeee..abdcef886 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -2,13 +2,14 @@ import argparse from datetime import date +import datetime import logging from django.core.management import BaseCommand from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper - +from datetime import datetime try: from epplib.exceptions import TransportError except ImportError: @@ -27,6 +28,7 @@ class Command(BaseCommand): self.update_success = [] self.update_skipped = [] self.update_failed = [] + self.expiration_cutoff = date(2023, 11, 15) def add_arguments(self, parser): """Add command line arguments.""" @@ -68,7 +70,7 @@ class Command(BaseCommand): self.check_if_positive_int(limit_parse, "limitParse") valid_domains = Domain.objects.filter( - expiration_date__gte=date(2023, 11, 15), state=Domain.State.READY + expiration_date__gte=self.expiration_cutoff, state=Domain.State.READY ).order_by("name") domains_to_change_count = valid_domains.count() @@ -81,49 +83,44 @@ class Command(BaseCommand): self.prompt_user_to_proceed(extension_amount, domains_to_change_count) for domain in valid_domains: - is_idempotent = self.idempotence_check(domain, extension_amount) - if not disable_idempotence and not is_idempotent: - self.update_skipped.append(domain.name) - logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") + try: + is_idempotent = self.idempotence_check(domain, extension_amount) + if not disable_idempotence and not is_idempotent: + self.update_skipped.append(domain.name) + logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") + else: + domain.renew_domain(extension_amount) + # Catches registry errors. Failures indicate bad data, or a faulty connection. + except (RegistryError, KeyError, TransportError) as err: + self.update_failed.append(domain.name) + logger.error( + f"{TerminalColors.FAIL}" f"Failed to update expiration date for {domain}" f"{TerminalColors.ENDC}" + ) + logger.error(err) else: - self.extend_expiration_date_on_domain(domain, extension_amount, debug) - - self.log_script_run_summary(debug) - - def extend_expiration_date_on_domain(self, domain: Domain, extension_amount: int, debug: bool): - """ - Given a particular domain, - extend the expiration date by the period specified in extension_amount - """ - try: - domain.renew_domain(extension_amount) - except (RegistryError, TransportError) as err: - logger.error( - f"{TerminalColors.FAIL}" f"Failed to update expiration date for {domain}" f"{TerminalColors.ENDC}" - ) - logger.error(err) - except Exception as err: - self.log_script_run_summary(debug) - raise err - else: - self.update_success.append(domain.name) - logger.info( - f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" - ) + self.update_success.append(domain.name) + logger.info( + f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" + ) + finally: + self.log_script_run_summary(debug) # == Helper functions == # - def idempotence_check(self, domain, extension_amount): + def idempotence_check(self, domain: Domain, extension_amount): """Determines if the proposed operation violates idempotency""" - proposed_date = self.add_years(domain.registry_expiration_date, extension_amount) # Because our migration data had a hard stop date, we can determine if our change - # is valid simply checking if adding two years to our current date yields a greater date - # than the proposed. - # CAVEAT: This check stops working after a year has elapsed between when this script - # was ran, and when it was ran again. This is good enough for now, but a more robust - # solution would be a DB flag. - extension_from_today = self.add_years(date.today(), extension_amount + 1) - is_idempotent = proposed_date < extension_from_today - return is_idempotent + # is valid simply checking the date is within a valid range and it was updated + # in epp on the current day. + # CAVEAT: This check stops working a day after it is ran (for some domains) and + # if the domain was updated by a user on the day it was ran. A more robust + # solution would be a db flag + proposed_date = self.add_years(domain.registry_expiration_date, extension_amount) + minimum_extension_date = self.add_years(self.expiration_cutoff, extension_amount) + maximum_extension_date = self.add_years(date(2025, 12, 31), extension_amount) + + valid_range = minimum_extension_date <= proposed_date <= maximum_extension_date + + return valid_range def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): """Asks if the user wants to proceed with this action""" From 9b78233b857e97244494af09a9be0c3b4989a43e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 09:48:21 -0700 Subject: [PATCH 26/78] Fix idempotent check --- .../commands/extend_expiration_dates.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index abdcef886..7372e2412 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -10,6 +10,8 @@ from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from datetime import datetime + +from registrar.models.transition_domain import TransitionDomain try: from epplib.exceptions import TransportError except ImportError: @@ -90,6 +92,10 @@ class Command(BaseCommand): logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") else: domain.renew_domain(extension_amount) + self.update_success.append(domain.name) + logger.info( + f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" + ) # Catches registry errors. Failures indicate bad data, or a faulty connection. except (RegistryError, KeyError, TransportError) as err: self.update_failed.append(domain.name) @@ -97,30 +103,30 @@ class Command(BaseCommand): f"{TerminalColors.FAIL}" f"Failed to update expiration date for {domain}" f"{TerminalColors.ENDC}" ) logger.error(err) - else: - self.update_success.append(domain.name) - logger.info( - f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" - ) - finally: + except Exception as err: self.log_script_run_summary(debug) + raise err + self.log_script_run_summary(debug) # == Helper functions == # def idempotence_check(self, domain: Domain, extension_amount): """Determines if the proposed operation violates idempotency""" # Because our migration data had a hard stop date, we can determine if our change # is valid simply checking the date is within a valid range and it was updated - # in epp on the current day. - # CAVEAT: This check stops working a day after it is ran (for some domains) and - # if the domain was updated by a user on the day it was ran. A more robust - # solution would be a db flag - proposed_date = self.add_years(domain.registry_expiration_date, extension_amount) + # in epp or not. + # CAVEAT: This is a workaround. A more robust solution would be a db flag + current_expiration_date = domain.registry_expiration_date + transition_domains = TransitionDomain.objects.filter( + domain_name=domain.name, + epp_expiration_date=current_expiration_date + ) + proposed_date = self.add_years(current_expiration_date, extension_amount) minimum_extension_date = self.add_years(self.expiration_cutoff, extension_amount) maximum_extension_date = self.add_years(date(2025, 12, 31), extension_amount) valid_range = minimum_extension_date <= proposed_date <= maximum_extension_date - return valid_range + return valid_range and transition_domains.count() > 0 def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): """Asks if the user wants to proceed with this action""" From 60344c3365f9476fe37662ed779744fbd1052c27 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Tue, 12 Dec 2023 12:13:04 -0600 Subject: [PATCH 27/78] Change email links to point to manage.get.gov and the correct object ID --- src/registrar/templates/emails/status_change_approved.txt | 4 ++-- src/registrar/templates/emails/status_change_in_review.txt | 2 +- src/registrar/templates/emails/submission_confirmation.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/templates/emails/status_change_approved.txt b/src/registrar/templates/emails/status_change_approved.txt index 80bf78842..25abe6e69 100644 --- a/src/registrar/templates/emails/status_change_approved.txt +++ b/src/registrar/templates/emails/status_change_approved.txt @@ -14,7 +14,7 @@ Now that your .gov domain has been approved, there are a few more things to do b YOU MUST ADD DOMAIN NAME SERVER INFORMATION Before your .gov domain can be used, you have to connect it to your Domain Name System (DNS) hosting service. At this time, we don’t provide DNS hosting services. -Go to the domain management page to add your domain name server information . +Go to the domain management page to add your domain name server information . Get help with adding your domain name server information . @@ -23,7 +23,7 @@ ADD DOMAIN MANAGERS, SECURITY EMAIL We strongly recommend that you add other points of contact who will help manage your domain. We also recommend that you provide a security email. This email will allow the public to report security issues on your domain. Security emails are made public. -Go to the domain management page to add domain contacts and a security email . +Go to the domain management page to add domain contacts and a security email . Get help with managing your .gov domain . diff --git a/src/registrar/templates/emails/status_change_in_review.txt b/src/registrar/templates/emails/status_change_in_review.txt index 63df669eb..b45b0d244 100644 --- a/src/registrar/templates/emails/status_change_in_review.txt +++ b/src/registrar/templates/emails/status_change_in_review.txt @@ -22,7 +22,7 @@ NEXT STEPS - We’re reviewing your request. This usually takes 20 business days. - You can check the status of your request at any time. - + - We’ll email you with questions or when we complete our review. diff --git a/src/registrar/templates/emails/submission_confirmation.txt b/src/registrar/templates/emails/submission_confirmation.txt index 41fab8005..2b0c0e86e 100644 --- a/src/registrar/templates/emails/submission_confirmation.txt +++ b/src/registrar/templates/emails/submission_confirmation.txt @@ -21,7 +21,7 @@ NEXT STEPS - We’ll review your request. This usually takes 20 business days. - You can check the status of your request at any time. - + - We’ll email you with questions or when we complete our review. From b2b08d0e6cf55f05ea6e08d1c42b82f904cd7e06 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:16:53 -0700 Subject: [PATCH 28/78] Fix test cases --- .../tests/test_transition_domain_migrations.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4efdee43e..eaa90bac3 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -29,14 +29,29 @@ class TestExtendExpirationDates(MockEppLib): Domain.objects.get_or_create( name="waterbutpurple.gov", state=Domain.State.READY, expiration_date=datetime.date(2023, 11, 15) ) + TransitionDomain.objects.get_or_create( + username="testytester@mail.com", + domain_name="waterbutpurple.gov", + expiration_date=datetime.date(2023, 11, 15) + ) # Create a domain with an invalid expiration date Domain.objects.get_or_create( name="fake.gov", state=Domain.State.READY, expiration_date=datetime.date(2022, 5, 25) ) + TransitionDomain.objects.get_or_create( + username="themoonisactuallycheese@mail.com", + domain_name="fake.gov", + expiration_date=datetime.date(2022, 5, 25) + ) # Create a domain with an invalid state Domain.objects.get_or_create( name="fakeneeded.gov", state=Domain.State.DNS_NEEDED, expiration_date=datetime.date(2023, 11, 15) ) + TransitionDomain.objects.get_or_create( + username="fakeneeded@mail.com", + domain_name="fakeneeded.gov", + expiration_date=datetime.date(2023, 11, 15) + ) def tearDown(self): """Deletes all DB objects related to migrations""" From 9cc9a8c2da7e2e9f699bde43a23639a45d16f3cd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:27:40 -0700 Subject: [PATCH 29/78] Typo --- src/registrar/tests/test_transition_domain_migrations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index eaa90bac3..d6b0eda0f 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -32,7 +32,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="testytester@mail.com", domain_name="waterbutpurple.gov", - expiration_date=datetime.date(2023, 11, 15) + epp_expiration_date=datetime.date(2023, 11, 15) ) # Create a domain with an invalid expiration date Domain.objects.get_or_create( @@ -41,7 +41,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="themoonisactuallycheese@mail.com", domain_name="fake.gov", - expiration_date=datetime.date(2022, 5, 25) + epp_expiration_date=datetime.date(2022, 5, 25) ) # Create a domain with an invalid state Domain.objects.get_or_create( @@ -50,7 +50,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="fakeneeded@mail.com", domain_name="fakeneeded.gov", - expiration_date=datetime.date(2023, 11, 15) + epp_expiration_date=datetime.date(2023, 11, 15) ) def tearDown(self): From 038fa3b266cb449ab14aabd34877f9ca8d90812d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:56:12 -0700 Subject: [PATCH 30/78] Fix test case + linting --- .../management/commands/extend_expiration_dates.py | 9 +++++---- src/registrar/tests/common.py | 14 +++++++++++++- .../tests/test_transition_domain_migrations.py | 14 ++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 7372e2412..4f9462a49 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -9,9 +9,9 @@ from django.core.management import BaseCommand from epplibwrapper.errors import RegistryError from registrar.models import Domain from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper -from datetime import datetime from registrar.models.transition_domain import TransitionDomain + try: from epplib.exceptions import TransportError except ImportError: @@ -94,7 +94,9 @@ class Command(BaseCommand): domain.renew_domain(extension_amount) self.update_success.append(domain.name) logger.info( - f"{TerminalColors.OKCYAN}" f"Successfully updated expiration date for {domain}" f"{TerminalColors.ENDC}" + f"{TerminalColors.OKCYAN}" + f"Successfully updated expiration date for {domain}" + f"{TerminalColors.ENDC}" ) # Catches registry errors. Failures indicate bad data, or a faulty connection. except (RegistryError, KeyError, TransportError) as err: @@ -117,8 +119,7 @@ class Command(BaseCommand): # CAVEAT: This is a workaround. A more robust solution would be a db flag current_expiration_date = domain.registry_expiration_date transition_domains = TransitionDomain.objects.filter( - domain_name=domain.name, - epp_expiration_date=current_expiration_date + domain_name=domain.name, epp_expiration_date=current_expiration_date ) proposed_date = self.add_years(current_expiration_date, extension_amount) minimum_extension_date = self.add_years(self.expiration_cutoff, extension_amount) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 5e1f324a7..337b6f31e 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -619,6 +619,17 @@ class MockEppLib(TestCase): ], ex_date=datetime.date(2023, 5, 25), ) + mockDataExtensionDomain = fakedEppObject( + "fakePw", + cr_date=datetime.datetime(2023, 5, 25, 19, 45, 35), + contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], + hosts=["fake.host.com"], + statuses=[ + common.Status(state="serverTransferProhibited", description="", lang="en"), + common.Status(state="inactive", description="", lang="en"), + ], + ex_date=datetime.date(2023, 11, 15), + ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) @@ -826,7 +837,7 @@ class MockEppLib(TestCase): mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", - ex_date=datetime.date(2025, 1, 10), + ex_date=datetime.date(2024, 11, 15), ) def _mockDomainName(self, _name, _avail=False): @@ -962,6 +973,7 @@ class MockEppLib(TestCase): self.infoDomainTwoHosts if self.mockedSendFunction.call_count == 5 else self.infoDomainNoHost, None, ), + "waterbutpurple.gov": (self.mockDataExtensionDomain, None), "nameserverwithip.gov": (self.infoDomainHasIP, None), "namerserversubdomain.gov": (self.infoDomainCheckHostIPCombo, None), "freeman.gov": (self.InfoDomainWithContacts, None), diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index d6b0eda0f..d3b81c42b 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -32,7 +32,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="testytester@mail.com", domain_name="waterbutpurple.gov", - epp_expiration_date=datetime.date(2023, 11, 15) + epp_expiration_date=datetime.date(2023, 11, 15), ) # Create a domain with an invalid expiration date Domain.objects.get_or_create( @@ -41,7 +41,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="themoonisactuallycheese@mail.com", domain_name="fake.gov", - epp_expiration_date=datetime.date(2022, 5, 25) + epp_expiration_date=datetime.date(2022, 5, 25), ) # Create a domain with an invalid state Domain.objects.get_or_create( @@ -50,7 +50,7 @@ class TestExtendExpirationDates(MockEppLib): TransitionDomain.objects.get_or_create( username="fakeneeded@mail.com", domain_name="fakeneeded.gov", - epp_expiration_date=datetime.date(2023, 11, 15) + epp_expiration_date=datetime.date(2023, 11, 15), ) def tearDown(self): @@ -84,16 +84,18 @@ class TestExtendExpirationDates(MockEppLib): Tests that the extend_expiration_dates method extends dates as expected """ desired_domain = Domain.objects.filter(name="waterbutpurple.gov").get() - desired_domain.expiration_date = datetime.date(2025, 1, 10) + desired_domain.expiration_date = datetime.date(2024, 11, 15) # Run the expiration date script self.run_extend_expiration_dates() current_domain = Domain.objects.filter(name="waterbutpurple.gov").get() + x = TransitionDomain.objects.filter(domain_name=current_domain.name).get() self.assertEqual(desired_domain, current_domain) - + print(f"wtf is going on {current_domain.__dict__}") + print(f"wtf is going on2 {x.__dict__}") # Explicitly test the expiration date - self.assertEqual(current_domain.expiration_date, datetime.date(2025, 1, 10)) + self.assertEqual(current_domain.expiration_date, datetime.date(2024, 11, 15)) def test_extends_expiration_date_skips_non_current(self): """ From 0d864c57efe2ee98d101fa4cafe2a06402564fde Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:56:57 -0700 Subject: [PATCH 31/78] Remove prints --- src/registrar/tests/test_transition_domain_migrations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index d3b81c42b..4bf9f09d3 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -92,8 +92,6 @@ class TestExtendExpirationDates(MockEppLib): current_domain = Domain.objects.filter(name="waterbutpurple.gov").get() x = TransitionDomain.objects.filter(domain_name=current_domain.name).get() self.assertEqual(desired_domain, current_domain) - print(f"wtf is going on {current_domain.__dict__}") - print(f"wtf is going on2 {x.__dict__}") # Explicitly test the expiration date self.assertEqual(current_domain.expiration_date, datetime.date(2024, 11, 15)) From 883008198f7b27382d98bfc8a7613b3e484f3997 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:01:52 -0700 Subject: [PATCH 32/78] Remove more testing debug stuff --- src/registrar/tests/test_transition_domain_migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4bf9f09d3..2da42462c 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -90,7 +90,7 @@ class TestExtendExpirationDates(MockEppLib): self.run_extend_expiration_dates() current_domain = Domain.objects.filter(name="waterbutpurple.gov").get() - x = TransitionDomain.objects.filter(domain_name=current_domain.name).get() + self.assertEqual(desired_domain, current_domain) # Explicitly test the expiration date self.assertEqual(current_domain.expiration_date, datetime.date(2024, 11, 15)) From 9f22ceb96606a47d9396fecaa2207403d27aa373 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:05:16 -0700 Subject: [PATCH 33/78] The final lint to lint them all --- src/registrar/management/commands/extend_expiration_dates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 4f9462a49..a9078b275 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -2,7 +2,6 @@ import argparse from datetime import date -import datetime import logging from django.core.management import BaseCommand From 319d980e989b638607a27752e95668db9e3dcc83 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 16:03:11 -0500 Subject: [PATCH 34/78] Clean up logs in the copy script, track eligible, processed and skipped --- .../copy_names_from_contacts_to_users.py | 181 ++++++++---------- 1 file changed, 79 insertions(+), 102 deletions(-) diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index 779788df9..038105f6a 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -25,16 +25,10 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - parser.add_argument( - "--limitParse", - default=0, - help="Sets max number of records (contacts) to copy, set to 0 to copy all entries", - ) - # ====================================================== # ===================== PRINTING ====================== # ====================================================== - def print_debug_mode_statements(self, debug_on: bool, debug_max_entries_to_parse: int): + def print_debug_mode_statements(self, debug_on: bool): """Prints additional terminal statements to indicate if --debug or --limitParse are in use""" TerminalHelper.print_conditional( @@ -45,45 +39,27 @@ class Command(BaseCommand): {TerminalColors.ENDC} """, ) - TerminalHelper.print_conditional( - debug_max_entries_to_parse > 0, - f"""{TerminalColors.OKCYAN} - ----------LIMITER ON---------- - Parsing of entries will be limited to - {debug_max_entries_to_parse} lines per file.") - Detailed print statements activated. - {TerminalColors.ENDC} - """, - ) - - def parse_limit_reached(self, debug_max_entries_to_parse: bool, total_rows_parsed: int) -> bool: - if debug_max_entries_to_parse > 0 and total_rows_parsed/2 >= debug_max_entries_to_parse: - logger.info( - f"""{TerminalColors.YELLOW} - ----PARSE LIMIT REACHED. HALTING PARSER.---- - {TerminalColors.ENDC} - """ - ) - return True - return False def print_summary_of_findings( self, - updated_user_records, skipped_contacts, + eligible_users, + processed_users, debug_on, ): """Prints to terminal a summary of findings from copying first and last names from contacts to users""" - total_updated_user_entries = len(updated_user_records) - total_skipped_contacts_entries = len(skipped_contacts) + total_eligible_users = len(eligible_users) + total_skipped_contacts = len(skipped_contacts) + total_processed_users = len(processed_users) logger.info( f"""{TerminalColors.OKGREEN} ============= FINISHED =============== - Updated {total_updated_user_entries} users - Skipped {total_skipped_contacts_entries} contacts + Skipped {total_skipped_contacts} contacts + Found {total_eligible_users} users linked to contacts + Processed {total_processed_users} users {TerminalColors.ENDC} """ # noqa ) @@ -93,11 +69,14 @@ class Command(BaseCommand): debug_on, f"""{TerminalColors.YELLOW} ======= DEBUG OUTPUT ======= - Updated User records: - {updated_user_records} + Users who have a linked contact: + {eligible_users} + + Processed users (users who have a linked contact and a missing first or last name): + {processed_users} ===== SKIPPED CONTACTS ===== - {skipped_contacts} + {skipped_contacts} {TerminalColors.ENDC} """, @@ -112,35 +91,35 @@ class Command(BaseCommand): Returns the corresponding User object. """ - - # Create some local variables to make data tracing easier - contact_email = contact.email - contact_first_name = contact.first_name - contact_lastname = contact.last_name - + user_exists = User.objects.filter(contact=contact).exists() if user_exists: try: # ----------------------- UPDATE USER ----------------------- # ---- GET THE USER - target_user = User.objects.get(contact=contact) + eligible_user = User.objects.get(contact=contact) + processed_user = None # DEBUG: TerminalHelper.print_conditional( debug_on, f"""{TerminalColors.YELLOW} - > Found linked entry in User table for: {contact_first_name} {contact_lastname} {contact_email} + > Found linked entry in User table for: + {contact.email} {contact.first_name} {contact.last_name} {TerminalColors.ENDC}""", # noqa ) # ---- UPDATE THE USER IF IT DOES NOT HAVE A FIRST AND LAST NAMES # ---- LET'S KEEP A LIGHT TOUCH - if not target_user.first_name or not target_user.last_name: - target_user.first_name = contact_first_name - target_user.last_name = contact_lastname - - target_user.save() + if not eligible_user.first_name or not eligible_user.last_name: + processed_user = eligible_user + processed_user.first_name = contact.first_name + processed_user.last_name = contact.last_name + processed_user.save() - return (target_user) + return ( + eligible_user, + processed_user, + ) except Exception as E: logger.warning( @@ -148,10 +127,14 @@ class Command(BaseCommand): {TerminalColors.FAIL} !!! ERROR: An exception occured in the User table for the following user: - {contact_email} + {contact.email} {contact.first_name} {contact.last_name} + + Exception is: {E} ----------TERMINATING----------""" ) sys.exit() + else: + return None, None # ====================================================== # ================= PROCESS CONTACTS ================== @@ -161,54 +144,49 @@ class Command(BaseCommand): def process_contacts( self, debug_on, - skipped_user_entries, - updated_user_entries, - debug_max_entries_to_parse, - total_rows_parsed, + skipped_contacts, + eligible_users, + processed_users, ): for contact in Contact.objects.all(): - # Create some local variables to make data tracing easier - contact_email = contact.email - contact_first_name = contact.first_name - contact_last_name = contact.last_name - + # DEBUG: - # TerminalHelper.print_conditional( - # debug_on, - # f"{TerminalColors.OKCYAN}" - # "Processing Contact: " - # f"{contact_email}," - # f" {contact_first_name}," - # f" {contact_last_name}" - # f"{TerminalColors.ENDC}", # noqa - # ) + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN}" + "Processing Contact: " + f"{contact.email}," + f" {contact.first_name}," + f" {contact.last_name}" + f"{TerminalColors.ENDC}", # noqa + ) # ====================================================== # ====================== USER ======================= - target_user = self.update_user(contact, debug_on) + (eligible_user, processed_user) = self.update_user(contact, debug_on) debug_string = "" - if target_user: + if eligible_user: # ---------------- UPDATED ---------------- - updated_user_entries.append(contact.email) - debug_string = f"updated user: {target_user}" + eligible_users.append(contact.email) + debug_string = f"eligible user: {eligible_user}" + if processed_user: + processed_users.append(contact.email) + debug_string = f"processed user: {processed_user}" else: - skipped_user_entries.append(contact.email) + skipped_contacts.append(contact.email) debug_string = f"skipped user: {contact.email}" # DEBUG: - # TerminalHelper.print_conditional( - # debug_on, - # (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), - # ) + TerminalHelper.print_conditional( + debug_on, + (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), + ) - # ------------------ Parse limit reached? ------------------ - # Check parse limit and exit loop if parse limit has been reached - if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): - break return ( - skipped_user_entries, - updated_user_entries, + skipped_contacts, + eligible_users, + processed_users, ) # ====================================================== @@ -224,19 +202,17 @@ class Command(BaseCommand): # grab command line arguments and store locally... debug_on = options.get("debug") - debug_max_entries_to_parse = int(options.get("limitParse")) # set to 0 to parse all entries - self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse) - - # users we UPDATED - updated_user_entries = [] + self.print_debug_mode_statements(debug_on) # users we SKIPPED - skipped_user_entries = [] - - # if we are limiting our parse (for testing purposes, keep - # track of total rows parsed) - total_rows_parsed = 0 + skipped_contacts = [] + + # users we found that are linked to contacts + eligible_users = [] + + # users we PROCESSED + processed_users = [] logger.info( f"""{TerminalColors.OKCYAN} @@ -252,18 +228,19 @@ class Command(BaseCommand): {TerminalColors.ENDC}""" ) ( - skipped_user_entries, - updated_user_entries, + skipped_contacts, + eligible_users, + processed_users, ) = self.process_contacts( debug_on, - skipped_user_entries, - updated_user_entries, - debug_max_entries_to_parse, - total_rows_parsed, + skipped_contacts, + eligible_users, + processed_users, ) self.print_summary_of_findings( - updated_user_entries, - skipped_user_entries, + skipped_contacts, + eligible_users, + processed_users, debug_on, ) From 80e1ad85e853906c9f23250cace5c418684b9732 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:46:56 -0700 Subject: [PATCH 35/78] Add migration --- .../0055_transitiondomain_processed.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/registrar/migrations/0055_transitiondomain_processed.py diff --git a/src/registrar/migrations/0055_transitiondomain_processed.py b/src/registrar/migrations/0055_transitiondomain_processed.py new file mode 100644 index 000000000..a2fb78edd --- /dev/null +++ b/src/registrar/migrations/0055_transitiondomain_processed.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.7 on 2023-12-12 21:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0054_alter_domainapplication_federal_agency_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="transitiondomain", + name="processed", + field=models.BooleanField( + default=True, + help_text="Indicates whether this TransitionDomain was already processed", + verbose_name="Processed", + ), + ), + ] From 8febad976d0a6e493b212dcfeb9319190fa5e589 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 17:26:31 -0500 Subject: [PATCH 36/78] WIP on script unit test --- .../copy_names_from_contacts_to_users.py | 5 +- .../test_copy_names_from_contacts_to_users.py | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 src/registrar/tests/test_copy_names_from_contacts_to_users.py diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index 038105f6a..c807c0278 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -103,8 +103,9 @@ class Command(BaseCommand): TerminalHelper.print_conditional( debug_on, f"""{TerminalColors.YELLOW} - > Found linked entry in User table for: - {contact.email} {contact.first_name} {contact.last_name} + > Found linked user for contact: + {contact} {contact.email} {contact.first_name} {contact.last_name} + > The linked user is {eligible_user} {TerminalColors.ENDC}""", # noqa ) diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py new file mode 100644 index 000000000..2383adc50 --- /dev/null +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -0,0 +1,92 @@ +from django.test import TestCase + +from registrar.models import ( + User, + Contact, +) + +from django.core.management import call_command +from unittest.mock import patch + +from registrar.management.commands.copy_names_from_contacts_to_users import Command + +class TestOrganizationMigration(TestCase): + def setUp(self): + """Defines the file name of migration_json and the folder its contained in""" + + + # self.user1, _ = User.objects.get_or_create(username="user1") + # self.user2 = User.objects.create(username="user2", first_name="Joey", last_name="") + # self.user3 = User.objects.create(username="user3", first_name="a special first name", last_name="a special last name") + # self.userX = User.objects.create(username="emailX@igorville.gov", first_name="firstX", last_name="lastX") + + # self.contact1, _ = Contact.objects.get_or_create(user=self.user1, email="email1@igorville.gov", first_name="first1", last_name="last1") + # self.contact2 = Contact.objects.create(user=self.user2, email="email2@igorville.gov", first_name="first2", last_name="last2") + # self.contact3 = Contact.objects.create(user=None, email="email3@igorville.gov", first_name="first3", last_name="last3") + # self.contact4 = Contact.objects.create(user=None, email="email4@igorville.gov", first_name="first4", last_name="last4") + + # self.contact1 = Contact.objects.create(email="email1@igorville.gov", first_name="first1", last_name="last1") + # self.contact2 = Contact.objects.create(email="email2@igorville.gov", first_name="first2", last_name="last2") + # self.contact3 = Contact.objects.create(email="email3@igorville.gov", first_name="first3", last_name="last3") + # self.contact4 = Contact.objects.create(email="email4@igorville.gov", first_name="first4", last_name="last4") + + # self.user1 = User.objects.create(contact=self.contact1) + # self.user2 = User.objects.create(contact=self.contact2, username="user2", first_name="Joey", last_name="") + # self.user3 = User.objects.create(username="user3", first_name="a special first name", last_name="a special last name") + # self.userX = User.objects.create(username="emailX@igorville.gov", first_name="firstX", last_name="lastX") + + + self.command = Command() + + def tearDown(self): + """Deletes all DB objects related to migrations""" + # Delete users + User.objects.all().delete() + Contact.objects.all().delete() + + def test_script_updates_linked_users(self): + + user1, _ = User.objects.get_or_create(username="user1") + contact1, _ = Contact.objects.get_or_create(user=user1, email="email1@igorville.gov", first_name="first1", last_name="last1") + + + # self.user1.first_name = "" + # self.user1.last_name = "" + # self.user2.last_name = "" + # self.user1.save() + # self.user2.save() + + # users we SKIPPED + skipped_contacts = [] + # users we found that are linked to contacts + eligible_users = [] + # users we PROCESSED + processed_users = [] + ( + skipped_contacts, + eligible_users, + processed_users, + ) = self.command.process_contacts( + True, + skipped_contacts, + eligible_users, + processed_users, + ) + + # self.user1.refresh_from_db() + # self.user2.refresh_from_db() + # self.user3.refresh_from_db() + # self.userX.refresh_from_db() + + self.assertEqual(user1.first_name, "first1") + self.assertEqual(user1.last_name, "last1") + # self.assertEqual(self.user2.first_name, "first2") + # self.assertEqual(self.user2.last_name, "last2") + # self.assertEqual(self.user3.first_name, "a special first name") + # self.assertEqual(self.user3.last_name, "a special last name") + # self.assertEqual(self.userX.first_name, "firstX") + # self.assertEqual(self.userX.last_name, "lastX") + + + + \ No newline at end of file From 26530f68bb59a6ff2d6159a64bd25384f5f0f115 Mon Sep 17 00:00:00 2001 From: Kristina Yin Date: Tue, 12 Dec 2023 15:07:40 -0800 Subject: [PATCH 37/78] updated all AO examples --- .../templates/includes/ao_example.html | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/registrar/templates/includes/ao_example.html b/src/registrar/templates/includes/ao_example.html index dfdbb3775..e0821fd39 100644 --- a/src/registrar/templates/includes/ao_example.html +++ b/src/registrar/templates/includes/ao_example.html @@ -1,50 +1,60 @@ {% if is_federal %} {% if federal_type == 'executive' %} -

Domain requests from executive branch agencies must be authorized by Chief Information Officers or agency heads.

-

Domain requests from executive branch agencies are subject to guidance issued by the U.S. Office of Management and Budget.

+

Executive branch federal agencies

+

Domain requests from executive branch federal agencies must be authorized by the agency's CIO or the head of the agency.

+

See OMB Memorandum M-23-10 for more information.

{% elif federal_type == 'judicial' %} -

Domain requests from the U.S. Supreme Court must be authorized by the director of information technology for the U.S. Supreme Court.

-

Domain requests from other judicial branch agencies must be authorized by the director or Chief Information Officer of the Administrative Office (AO) of the United States Courts. -

+

Judicial branch federal agencies

+

Domain requests for judicial branch federal agencies, except the U.S. Supreme Court, must be authorized by the director or CIO of the Administrative Office (AO) of the United States Courts.

+

Domain requests from the U.S. Supreme Court must be authorized by the director of information technology for the U.S. Supreme Court.

{% elif federal_type == 'legislative' %} -

U.S. Senate

-

Domain requests from the U.S. Senate must come from the Senate Sergeant at Arms.

+

Legislative branch federal agencies

-

U.S. House of Representatives

-

Domain requests from the U.S. House of Representatives must come from the House Chief Administrative Officer. +

U.S. Senate

+

Domain requests from the U.S. Senate must come from the Senate Sergeant at Arms.

-

Other legislative branch agencies

-

Domain requests from legislative branch agencies must come from the agency’s head or Chief Information Officer.

-

Domain requests from legislative commissions must come from the head of the commission, or the head or Chief Information Officer of the parent agency, if there is one. -

+

U.S. House of Representatives

+

Domain requests from the U.S. House of Representatives must come from the House Chief Administrative Officer.

+ +

Other legislative branch agencies

+

Domain requests from legislative branch agencies must come from the agency’s head or CIO.

+

Domain requests from legislative commissions must come from the head of the commission, or the head or CIO of the parent agency, if there is one.

{% endif %} {% elif organization_type == 'city' %} -

Domain requests from cities must be authorized by the mayor or the equivalent highest-elected official.

+

Cities

+

Domain requests from cities must be authorized by someone in a role of significant, executive responsibility within the city (mayor, council president, city manager, township/village supervisor, select board chairperson, chief, senior technology officer, or equivalent).

{% elif organization_type == 'county' %} -

Domain requests from counties must be authorized by the chair of the county commission or the equivalent highest-elected official.

+

Counties

+

Domain requests from counties must be authorized by the commission chair or someone in a role of significant, executive responsibility within the county (county judge, county mayor, parish/borough president, senior technology officer, or equivalent). Other county-level offices (county clerk, sheriff, county auditor, comptroller) may qualify, as well, in some instances.

{% elif organization_type == 'interstate' %} -

Domain requests from interstate organizations must be authorized by the highest-ranking executive (president, director, chair, or equivalent) or one of the state’s governors or Chief Information Officers.

+

Interstate organizations

+

Domain requests from interstate organizations must be authorized by someone in a role of significant, executive responsibility within the organization (president, director, chair, senior technology officer, or equivalent) or one of the state’s governors or CIOs.

{% elif organization_type == 'school_district' %} +

School districts

Domain requests from school district governments must be authorized by the highest-ranking executive (the chair of a school district’s board or a superintendent).

{% elif organization_type == 'special_district' %} -

Domain requests from special districts must be authorized by the highest-ranking executive (president, director, chair, or equivalent) or state Chief Information Officers for state-based organizations.

+

Special districts

+

Domain requests from school district governments must be authorized by someone in a role of significant, executive responsibility within the district (board chair, superintendent, senior technology officer, or equivalent).

{% elif organization_type == 'state_or_territory' %} -

States and territories: executive branch

-

Domain requests from states and territories must be authorized by the governor or the state Chief Information Officer.

-

States and territories: judicial and legislative branches

-

Domain requests from state legislatures and courts must be authorized by an agency’s Chief Information Officer or highest-ranking executive.

+

U.S. states and territories

+ +

States and territories: executive branch

+

Domain requests from states and territories must be authorized by the governor or someone in a role of significant, executive responsibility within the agency (department secretary, senior technology officer, or equivalent).

+ +

States and territories: judicial and legislative branches

+

Domain requests from state legislatures and courts must be authorized by an agency’s CIO or someone in a role of significant, executive responsibility within the agency.

{% elif organization_type == 'tribal' %} -

Domain requests from federally-recognized tribal governments must be authorized by the leader of the tribe, as recognized by the Bureau of Indian Affairs.

-

Domain requests from state-recognized tribal governments must be authorized by the leader of the tribe, as determined by the state’s tribal recognition initiative.

+

Tribal governments

+

Domain requests from federally recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

{% endif %} From e7e3df042243443226c01dfa637d4449e538b058 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 18:28:25 -0500 Subject: [PATCH 38/78] Lint --- src/djangooidc/backends.py | 2 +- src/djangooidc/tests/test_backends.py | 8 +- src/djangooidc/views.py | 19 +-- .../copy_names_from_contacts_to_users.py | 19 +-- .../test_copy_names_from_contacts_to_users.py | 136 +++++++++--------- src/registrar/tests/test_models.py | 22 ++- 6 files changed, 107 insertions(+), 99 deletions(-) diff --git a/src/djangooidc/backends.py b/src/djangooidc/backends.py index d77a3fe15..cf326eca4 100644 --- a/src/djangooidc/backends.py +++ b/src/djangooidc/backends.py @@ -69,7 +69,7 @@ class OpenIdConnectBackend(ModelBackend): # Overwrite first_name and last_name if not empty string for key, value in kwargs.items(): # Check if the key is not first_name or last_name or value is not empty string - if key not in ['first_name', 'last_name'] or value: + if key not in ["first_name", "last_name"] or value: setattr(user, key, value) user.save() diff --git a/src/djangooidc/tests/test_backends.py b/src/djangooidc/tests/test_backends.py index 51fc965c4..93dc6e68a 100644 --- a/src/djangooidc/tests/test_backends.py +++ b/src/djangooidc/tests/test_backends.py @@ -4,8 +4,8 @@ from django.utils import timezone from registrar.models import User from ..backends import OpenIdConnectBackend # Adjust the import path based on your project structure -class OpenIdConnectBackendTestCase(TestCase): +class OpenIdConnectBackendTestCase(TestCase): def setUp(self): self.backend = OpenIdConnectBackend() self.kwargs = { @@ -56,10 +56,10 @@ class OpenIdConnectBackendTestCase(TestCase): """Test that authenticate updates an existing user if it finds one. For this test, given_name and family_name are not supplied""" # Create an existing user with the same username and with first and last names - existing_user = User.objects.create_user(username="test_user",first_name="John",last_name="Doe") + existing_user = User.objects.create_user(username="test_user", first_name="John", last_name="Doe") # Remove given_name and family_name from the input, self.kwargs - self.kwargs.pop("given_name", None) + self.kwargs.pop("given_name", None) self.kwargs.pop("family_name", None) # Ensure that the authenticate method updates the existing user @@ -79,7 +79,7 @@ class OpenIdConnectBackendTestCase(TestCase): """Test that authenticate updates an existing user if it finds one. For this test, given_name and family_name are supplied and overwrite""" # Create an existing user with the same username and with first and last names - existing_user = User.objects.create_user(username="test_user",first_name="WillBe",last_name="Replaced") + existing_user = User.objects.create_user(username="test_user", first_name="WillBe", last_name="Replaced") # Ensure that the authenticate method updates the existing user # and preserves existing first and last names diff --git a/src/djangooidc/views.py b/src/djangooidc/views.py index 7efb576da..a39da68aa 100644 --- a/src/djangooidc/views.py +++ b/src/djangooidc/views.py @@ -58,7 +58,7 @@ def openid(request): request.session["next"] = request.GET.get("next", "/") try: - logger.info('openid() calls create_authn_request in oidc') + logger.info("openid() calls create_authn_request in oidc") return CLIENT.create_authn_request(request.session) except Exception as err: return error_page(request, err) @@ -72,22 +72,23 @@ def login_callback(request): # test for need for identity verification and if it is satisfied # if not satisfied, redirect user to login with stepped up acr_value if requires_step_up_auth(userinfo): - logger.info('login_callback() calls get_step_up_acr_value and create_authn_request in oidc') + logger.info("login_callback() calls get_step_up_acr_value and create_authn_request in oidc") # add acr_value to request.session request.session["acr_value"] = CLIENT.get_step_up_acr_value() return CLIENT.create_authn_request(request.session) - - logger.info(f'login_callback() before calling authenticate: {userinfo}') - + + logger.info(f"login_callback() before calling authenticate: {userinfo}") + try: user_in_db = User.objects.get(username=userinfo["sub"]) - + if user_in_db: - logger.info(f"This user exists in the DB (before authenticate): {user_in_db.first_name} {user_in_db.last_name}") + logger.info( + f"This user exists in the DB (before authenticate): {user_in_db.first_name} {user_in_db.last_name}" + ) except: pass - - + user = authenticate(request=request, **userinfo) if user: login(request, user) diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index c807c0278..6fdf0f09c 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -63,7 +63,7 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ # noqa ) - + # DEBUG: TerminalHelper.print_conditional( debug_on, @@ -91,7 +91,7 @@ class Command(BaseCommand): Returns the corresponding User object. """ - + user_exists = User.objects.filter(contact=contact).exists() if user_exists: try: @@ -105,7 +105,7 @@ class Command(BaseCommand): f"""{TerminalColors.YELLOW} > Found linked user for contact: {contact} {contact.email} {contact.first_name} {contact.last_name} - > The linked user is {eligible_user} + > The linked user is {eligible_user} {eligible_user.username} {TerminalColors.ENDC}""", # noqa ) @@ -113,8 +113,10 @@ class Command(BaseCommand): # ---- LET'S KEEP A LIGHT TOUCH if not eligible_user.first_name or not eligible_user.last_name: processed_user = eligible_user - processed_user.first_name = contact.first_name - processed_user.last_name = contact.last_name + # (expression has type "str | None", variable has type "str | int | Combinable") + # so we'll ignore type + processed_user.first_name = contact.first_name # type: ignore + processed_user.last_name = contact.last_name # type: ignore processed_user.save() return ( @@ -140,7 +142,7 @@ class Command(BaseCommand): # ====================================================== # ================= PROCESS CONTACTS ================== # ====================================================== - + # C901 'Command.handle' is too complex def process_contacts( self, @@ -150,7 +152,6 @@ class Command(BaseCommand): processed_users, ): for contact in Contact.objects.all(): - # DEBUG: TerminalHelper.print_conditional( debug_on, @@ -208,10 +209,10 @@ class Command(BaseCommand): # users we SKIPPED skipped_contacts = [] - + # users we found that are linked to contacts eligible_users = [] - + # users we PROCESSED processed_users = [] diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py index 2383adc50..eaf0e6349 100644 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -5,88 +5,96 @@ from registrar.models import ( Contact, ) -from django.core.management import call_command -from unittest.mock import patch - from registrar.management.commands.copy_names_from_contacts_to_users import Command -class TestOrganizationMigration(TestCase): + +class TestDataUpdates(TestCase): def setUp(self): - """Defines the file name of migration_json and the folder its contained in""" - - - # self.user1, _ = User.objects.get_or_create(username="user1") - # self.user2 = User.objects.create(username="user2", first_name="Joey", last_name="") - # self.user3 = User.objects.create(username="user3", first_name="a special first name", last_name="a special last name") - # self.userX = User.objects.create(username="emailX@igorville.gov", first_name="firstX", last_name="lastX") - - # self.contact1, _ = Contact.objects.get_or_create(user=self.user1, email="email1@igorville.gov", first_name="first1", last_name="last1") - # self.contact2 = Contact.objects.create(user=self.user2, email="email2@igorville.gov", first_name="first2", last_name="last2") - # self.contact3 = Contact.objects.create(user=None, email="email3@igorville.gov", first_name="first3", last_name="last3") - # self.contact4 = Contact.objects.create(user=None, email="email4@igorville.gov", first_name="first4", last_name="last4") - - # self.contact1 = Contact.objects.create(email="email1@igorville.gov", first_name="first1", last_name="last1") - # self.contact2 = Contact.objects.create(email="email2@igorville.gov", first_name="first2", last_name="last2") - # self.contact3 = Contact.objects.create(email="email3@igorville.gov", first_name="first3", last_name="last3") - # self.contact4 = Contact.objects.create(email="email4@igorville.gov", first_name="first4", last_name="last4") - - # self.user1 = User.objects.create(contact=self.contact1) - # self.user2 = User.objects.create(contact=self.contact2, username="user2", first_name="Joey", last_name="") - # self.user3 = User.objects.create(username="user3", first_name="a special first name", last_name="a special last name") - # self.userX = User.objects.create(username="emailX@igorville.gov", first_name="firstX", last_name="lastX") - - + """We cannot setup the user details because contacts will override the first and last names in its save method + so we will initiate the users, setup the contacts and link them, and leave the rest of the setup for the test(s). + """ + + self.user1 = User.objects.create(username="user1") + self.user2 = User.objects.create(username="user2") + self.user3 = User.objects.create(username="user3") + self.userX = User.objects.create(username="user4") + # The last user created triggers the creation of a contact and attaches itself to it. @Neil wth is going on? + # This bs_user defuses that situation so we can test the code. + self.bs_user = User.objects.create() + + self.contact1 = Contact.objects.create( + user=self.user1, email="email1@igorville.gov", first_name="first1", last_name="last1" + ) + self.contact2 = Contact.objects.create( + user=self.user2, email="email2@igorville.gov", first_name="first2", last_name="last2" + ) + self.contact3 = Contact.objects.create( + user=self.user3, email="email3@igorville.gov", first_name="first3", last_name="last3" + ) + self.contact4 = Contact.objects.create(email="email4@igorville.gov", first_name="first4", last_name="last4") + self.command = Command() def tearDown(self): - """Deletes all DB objects related to migrations""" - # Delete users + """Clean up""" + # Delete users and contacts User.objects.all().delete() Contact.objects.all().delete() - + def test_script_updates_linked_users(self): - - user1, _ = User.objects.get_or_create(username="user1") - contact1, _ = Contact.objects.get_or_create(user=user1, email="email1@igorville.gov", first_name="first1", last_name="last1") - - - # self.user1.first_name = "" - # self.user1.last_name = "" - # self.user2.last_name = "" - # self.user1.save() - # self.user2.save() - - # users we SKIPPED + """Test the script that copies contacts' first and last names into associated users that + are eligible (first or last are blank or undefined)""" + + # Set up the users' first and last names here so + # they that they don't get overwritten by Contact's save() + # User with no first or last names + self.user1.first_name = "" + self.user1.last_name = "" + self.user1.save() + + # User with a first name but no last name + self.user2.last_name = "" + self.user2.save() + + # User with a first and last name + self.user3.first_name = "An existing first name" + self.user3.last_name = "An existing last name" + self.user3.save() + + # Unlinked user + # To make this test useful, we will set the last_name to "" + self.userX.first_name = "Unlinked user's first name" + self.userX.last_name = "" + self.userX.save() + + # Call the parent method the same way we do it in the script skipped_contacts = [] - # users we found that are linked to contacts eligible_users = [] - # users we PROCESSED processed_users = [] ( skipped_contacts, eligible_users, processed_users, ) = self.command.process_contacts( - True, + # Set debugging to False + False, skipped_contacts, eligible_users, processed_users, ) - - # self.user1.refresh_from_db() - # self.user2.refresh_from_db() - # self.user3.refresh_from_db() - # self.userX.refresh_from_db() - - self.assertEqual(user1.first_name, "first1") - self.assertEqual(user1.last_name, "last1") - # self.assertEqual(self.user2.first_name, "first2") - # self.assertEqual(self.user2.last_name, "last2") - # self.assertEqual(self.user3.first_name, "a special first name") - # self.assertEqual(self.user3.last_name, "a special last name") - # self.assertEqual(self.userX.first_name, "firstX") - # self.assertEqual(self.userX.last_name, "lastX") - - - - \ No newline at end of file + + # Trigger DB refresh + self.user1.refresh_from_db() + self.user2.refresh_from_db() + self.user3.refresh_from_db() + self.userX.refresh_from_db() + + # Asserts + self.assertEqual(self.user1.first_name, "first1") + self.assertEqual(self.user1.last_name, "last1") + self.assertEqual(self.user2.first_name, "first2") + self.assertEqual(self.user2.last_name, "last2") + self.assertEqual(self.user3.first_name, "An existing first name") + self.assertEqual(self.user3.last_name, "An existing last name") + self.assertEqual(self.userX.first_name, "Unlinked user's first name") + self.assertEqual(self.userX.last_name, "") diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index c51af8008..dfab4f9a3 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -654,10 +654,9 @@ class TestUser(TestCase): """A new user who's neither transitioned nor invited should return True when tested with class method needs_identity_verification""" self.assertTrue(User.needs_identity_verification(self.user.email, self.user.username)) - + class TestContact(TestCase): - def setUp(self): self.email = "mayor@igorville.gov" self.user, _ = User.objects.get_or_create(email=self.email, first_name="Rachid", last_name="Mrad") @@ -675,29 +674,28 @@ class TestContact(TestCase): self.assertEqual(self.contact.last_name, "Mrad") self.assertEqual(self.user.first_name, "Rachid") self.assertEqual(self.user.last_name, "Mrad") - + self.contact.first_name = "Joey" self.contact.last_name = "Baloney" self.contact.save() - + # Refresh the user object to reflect the changes made in the database self.user.refresh_from_db() - - # Updating the contact's first and last names propagate to the user + + # Updating the contact's first and last names propagate to the user self.assertEqual(self.contact.first_name, "Joey") self.assertEqual(self.contact.last_name, "Baloney") self.assertEqual(self.user.first_name, "Joey") self.assertEqual(self.user.last_name, "Baloney") - + def test_saving_contact_does_not_update_user_email(self): - """When a contact's email is updated, the change is not propagated to the lined user.""" + """When a contact's email is updated, the change is not propagated to the lined user.""" self.contact.email = "joey.baloney@diaperville.com" self.contact.save() - + # Refresh the user object to reflect the changes made in the database self.user.refresh_from_db() - - # Updating the contact's email does not propagate + + # Updating the contact's email does not propagate self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") self.assertEqual(self.user.email, "mayor@igorville.gov") - From 8b6ce47da18dbe7a22337c94ce3a0700ec5025d3 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 18:52:12 -0500 Subject: [PATCH 39/78] Changed the logic and test around users who have either a first or a last: Left them alone instead of overwriting from contact --- .../copy_names_from_contacts_to_users.py | 2 +- .../test_copy_names_from_contacts_to_users.py | 22 +++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index 6fdf0f09c..f4849b2bd 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -111,7 +111,7 @@ class Command(BaseCommand): # ---- UPDATE THE USER IF IT DOES NOT HAVE A FIRST AND LAST NAMES # ---- LET'S KEEP A LIGHT TOUCH - if not eligible_user.first_name or not eligible_user.last_name: + if not eligible_user.first_name and not eligible_user.last_name: processed_user = eligible_user # (expression has type "str | None", variable has type "str | int | Combinable") # so we'll ignore type diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py index eaf0e6349..4a57ec19b 100644 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -17,7 +17,7 @@ class TestDataUpdates(TestCase): self.user1 = User.objects.create(username="user1") self.user2 = User.objects.create(username="user2") self.user3 = User.objects.create(username="user3") - self.userX = User.objects.create(username="user4") + self.user4 = User.objects.create(username="user4") # The last user created triggers the creation of a contact and attaches itself to it. @Neil wth is going on? # This bs_user defuses that situation so we can test the code. self.bs_user = User.objects.create() @@ -53,6 +53,7 @@ class TestDataUpdates(TestCase): self.user1.save() # User with a first name but no last name + self.user2.first_name = "First name but no last name" self.user2.last_name = "" self.user2.save() @@ -61,12 +62,6 @@ class TestDataUpdates(TestCase): self.user3.last_name = "An existing last name" self.user3.save() - # Unlinked user - # To make this test useful, we will set the last_name to "" - self.userX.first_name = "Unlinked user's first name" - self.userX.last_name = "" - self.userX.save() - # Call the parent method the same way we do it in the script skipped_contacts = [] eligible_users = [] @@ -87,14 +82,17 @@ class TestDataUpdates(TestCase): self.user1.refresh_from_db() self.user2.refresh_from_db() self.user3.refresh_from_db() - self.userX.refresh_from_db() # Asserts + # The user that has no first and last names will get them from the contact self.assertEqual(self.user1.first_name, "first1") self.assertEqual(self.user1.last_name, "last1") - self.assertEqual(self.user2.first_name, "first2") - self.assertEqual(self.user2.last_name, "last2") + # The user that has a first but no last will be left alone + self.assertEqual(self.user2.first_name, "First name but no last name") + self.assertEqual(self.user2.last_name, "") + # The user that has a first and a last will be left alone self.assertEqual(self.user3.first_name, "An existing first name") self.assertEqual(self.user3.last_name, "An existing last name") - self.assertEqual(self.userX.first_name, "Unlinked user's first name") - self.assertEqual(self.userX.last_name, "") + # The unlinked user will be left alone + self.assertEqual(self.user4.first_name, "") + self.assertEqual(self.user4.last_name, "") From 5254b7cda8818b41b90c7726ae71174673604f77 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 19:27:45 -0500 Subject: [PATCH 40/78] Logger cleanup --- src/djangooidc/oidc.py | 16 ++++++++-------- src/djangooidc/views.py | 14 -------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/djangooidc/oidc.py b/src/djangooidc/oidc.py index 331490cae..87592d8e1 100644 --- a/src/djangooidc/oidc.py +++ b/src/djangooidc/oidc.py @@ -87,7 +87,7 @@ class Client(oic.Client): extra_args=None, ): """Step 2: Construct a login URL at OP's domain and send the user to it.""" - logger.info("create_authn_request() Creating the OpenID Connect authn request...") + logger.debug("create_authn_request() Creating the OpenID Connect authn request...") state = rndstr(size=32) try: session["state"] = state @@ -112,7 +112,7 @@ class Client(oic.Client): logger.error("Failed to assemble request arguments for %s" % state) raise o_e.InternalError(locator=state) - logger.info("request args: %s" % request_args) + logger.debug("request args: %s" % request_args) try: # prepare the request for sending @@ -126,9 +126,9 @@ class Client(oic.Client): method="GET", request_args=request_args, ) - logger.info("body: %s" % body) - logger.info("URL: %s" % url) - logger.info("headers: %s" % headers) + logger.debug("body: %s" % body) + logger.debug("URL: %s" % url) + logger.debug("headers: %s" % headers) except Exception as err: logger.error(err) logger.error("Failed to prepare request for %s" % state) @@ -150,7 +150,7 @@ class Client(oic.Client): def callback(self, unparsed_response, session): """Step 3: Receive OP's response, request an access token, and user info.""" - logger.info("callback() Processing the OpenID Connect callback response...") + logger.debug("callback() Processing the OpenID Connect callback response...") state = session.get("state", "") try: # parse the response from OP @@ -174,7 +174,7 @@ class Client(oic.Client): logger.error("Unable to process response %s for %s" % (error, state)) raise o_e.AuthenticationFailed(locator=state) - logger.info("callback() authn_response %s" % authn_response) + logger.debug("callback() authn_response %s" % authn_response) if not authn_response.get("state", None): logger.error("State value not received from OP for %s" % state) @@ -213,7 +213,7 @@ class Client(oic.Client): logger.error("Unable to get user info (%s) for %s" % (info_response.get("error", ""), state)) raise o_e.AuthenticationFailed(locator=state) - logger.info("_get_user_info() user info: %s" % info_response) + logger.debug("_get_user_info() user info: %s" % info_response) return info_response.to_dict() diff --git a/src/djangooidc/views.py b/src/djangooidc/views.py index a39da68aa..e7151d8a3 100644 --- a/src/djangooidc/views.py +++ b/src/djangooidc/views.py @@ -72,23 +72,9 @@ def login_callback(request): # test for need for identity verification and if it is satisfied # if not satisfied, redirect user to login with stepped up acr_value if requires_step_up_auth(userinfo): - logger.info("login_callback() calls get_step_up_acr_value and create_authn_request in oidc") # add acr_value to request.session request.session["acr_value"] = CLIENT.get_step_up_acr_value() return CLIENT.create_authn_request(request.session) - - logger.info(f"login_callback() before calling authenticate: {userinfo}") - - try: - user_in_db = User.objects.get(username=userinfo["sub"]) - - if user_in_db: - logger.info( - f"This user exists in the DB (before authenticate): {user_in_db.first_name} {user_in_db.last_name}" - ) - except: - pass - user = authenticate(request=request, **userinfo) if user: login(request, user) From 2f47272fff93c4cf03ad1c97bb67cc0dbde1cc38 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 19:28:38 -0500 Subject: [PATCH 41/78] Logger cleanup --- src/djangooidc/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/djangooidc/views.py b/src/djangooidc/views.py index e7151d8a3..b5905df48 100644 --- a/src/djangooidc/views.py +++ b/src/djangooidc/views.py @@ -58,7 +58,6 @@ def openid(request): request.session["next"] = request.GET.get("next", "/") try: - logger.info("openid() calls create_authn_request in oidc") return CLIENT.create_authn_request(request.session) except Exception as err: return error_page(request, err) From 18e215dd084f2837cf1f536037a7d64a87312a73 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 19:31:09 -0500 Subject: [PATCH 42/78] Logger cleanup --- src/djangooidc/oidc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/djangooidc/oidc.py b/src/djangooidc/oidc.py index 87592d8e1..0c5f2c637 100644 --- a/src/djangooidc/oidc.py +++ b/src/djangooidc/oidc.py @@ -87,7 +87,7 @@ class Client(oic.Client): extra_args=None, ): """Step 2: Construct a login URL at OP's domain and send the user to it.""" - logger.debug("create_authn_request() Creating the OpenID Connect authn request...") + logger.debug("Creating the OpenID Connect authn request...") state = rndstr(size=32) try: session["state"] = state @@ -150,7 +150,7 @@ class Client(oic.Client): def callback(self, unparsed_response, session): """Step 3: Receive OP's response, request an access token, and user info.""" - logger.debug("callback() Processing the OpenID Connect callback response...") + logger.debug("Processing the OpenID Connect callback response...") state = session.get("state", "") try: # parse the response from OP @@ -174,7 +174,7 @@ class Client(oic.Client): logger.error("Unable to process response %s for %s" % (error, state)) raise o_e.AuthenticationFailed(locator=state) - logger.debug("callback() authn_response %s" % authn_response) + logger.debug("authn_response %s" % authn_response) if not authn_response.get("state", None): logger.error("State value not received from OP for %s" % state) @@ -213,7 +213,7 @@ class Client(oic.Client): logger.error("Unable to get user info (%s) for %s" % (info_response.get("error", ""), state)) raise o_e.AuthenticationFailed(locator=state) - logger.debug("_get_user_info() user info: %s" % info_response) + logger.debug("user info: %s" % info_response) return info_response.to_dict() From 7c5eb79f0a2a2b38948f191d3e711554bfb90095 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 19:31:36 -0500 Subject: [PATCH 43/78] cleanup --- src/djangooidc/oidc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/djangooidc/oidc.py b/src/djangooidc/oidc.py index 0c5f2c637..91bfddc66 100644 --- a/src/djangooidc/oidc.py +++ b/src/djangooidc/oidc.py @@ -214,7 +214,6 @@ class Client(oic.Client): raise o_e.AuthenticationFailed(locator=state) logger.debug("user info: %s" % info_response) - return info_response.to_dict() def _request_token(self, state, code, session): From a422a7cbdb8e00e0ff796cddb71774666e51b7e2 Mon Sep 17 00:00:00 2001 From: Kristina Yin Date: Tue, 12 Dec 2023 16:33:05 -0800 Subject: [PATCH 44/78] updated python test since first line of ao_example.html was changed --- src/registrar/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 3c98c5fe7..da6fe6205 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -804,7 +804,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # ---- AO CONTACT PAGE ---- self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) ao_page = org_contact_result.follow() - self.assertContains(ao_page, "Domain requests from executive branch agencies") + self.assertContains(ao_page, "Executive branch federal agencies") # Go back to organization type page and change type self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) From 9b90172cd31b59d752ad5c6856a2e0eaf69acc0b Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 20:26:46 -0500 Subject: [PATCH 45/78] lint --- src/djangooidc/tests/test_backends.py | 2 -- .../commands/copy_names_from_contacts_to_users.py | 6 +++--- .../tests/test_copy_names_from_contacts_to_users.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/djangooidc/tests/test_backends.py b/src/djangooidc/tests/test_backends.py index 93dc6e68a..ac7f74903 100644 --- a/src/djangooidc/tests/test_backends.py +++ b/src/djangooidc/tests/test_backends.py @@ -1,6 +1,4 @@ from django.test import TestCase -from django.contrib.auth import get_user_model -from django.utils import timezone from registrar.models import User from ..backends import OpenIdConnectBackend # Adjust the import path based on your project structure diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index f4849b2bd..c5e2364e9 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -71,10 +71,10 @@ class Command(BaseCommand): ======= DEBUG OUTPUT ======= Users who have a linked contact: {eligible_users} - + Processed users (users who have a linked contact and a missing first or last name): {processed_users} - + ===== SKIPPED CONTACTS ===== {skipped_contacts} @@ -131,7 +131,7 @@ class Command(BaseCommand): !!! ERROR: An exception occured in the User table for the following user: {contact.email} {contact.first_name} {contact.last_name} - + Exception is: {E} ----------TERMINATING----------""" ) diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py index 4a57ec19b..2690578e0 100644 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -11,7 +11,7 @@ from registrar.management.commands.copy_names_from_contacts_to_users import Comm class TestDataUpdates(TestCase): def setUp(self): """We cannot setup the user details because contacts will override the first and last names in its save method - so we will initiate the users, setup the contacts and link them, and leave the rest of the setup for the test(s). + so we will initiate the users, setup the contacts and link them, and leave the rest of the setup to the test(s). """ self.user1 = User.objects.create(username="user1") From 2c5d6c8488ef4da4789784e1e9083bd29dc652a5 Mon Sep 17 00:00:00 2001 From: Kristina Yin Date: Tue, 12 Dec 2023 17:34:09 -0800 Subject: [PATCH 46/78] accidently mixing up special district and school district descriptions --- src/registrar/templates/includes/ao_example.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/templates/includes/ao_example.html b/src/registrar/templates/includes/ao_example.html index e0821fd39..976deb34d 100644 --- a/src/registrar/templates/includes/ao_example.html +++ b/src/registrar/templates/includes/ao_example.html @@ -38,11 +38,12 @@ {% elif organization_type == 'school_district' %}

School districts

-

Domain requests from school district governments must be authorized by the highest-ranking executive (the chair of a school district’s board or a superintendent).

+

Domain requests from school district governments must be authorized by someone in a role of significant, executive responsibility within the district (board chair, superintendent, senior technology officer, or equivalent).

{% elif organization_type == 'special_district' %}

Special districts

-

Domain requests from school district governments must be authorized by someone in a role of significant, executive responsibility within the district (board chair, superintendent, senior technology officer, or equivalent).

+

Domain requests from special districts must be authorized by someone in a role of significant, executive responsibility within the district (CEO, chair, executive director, senior technology officer, or equivalent). +

{% elif organization_type == 'state_or_territory' %}

U.S. states and territories

From 600f326be23639ee3ee9e02a812846c7dadd5a8d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 21:13:42 -0500 Subject: [PATCH 47/78] Make username in dummy_user unique and comment out parts in test_alphabetically_sorted_fk_fields_domain_application and test_alphabetically_sorted_fk_fields_domain_information that now fail after that edit --- src/registrar/tests/common.py | 21 +++++++++++---------- src/registrar/tests/test_admin.py | 6 +++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index f17e0f9fa..347781ef8 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -5,6 +5,7 @@ import logging from contextlib import contextmanager import random from string import ascii_uppercase +import uuid from django.test import TestCase from unittest.mock import MagicMock, Mock, patch from typing import List, Dict @@ -155,16 +156,6 @@ class AuditedAdminMockData: APPLICATION = "application" INVITATION = "invitation" - def dummy_user(self, item_name, short_hand): - """Creates a dummy user object, - but with a shorthand and support for multiple""" - user = User.objects.get_or_create( - first_name="{} first_name:{}".format(item_name, short_hand), - last_name="{} last_name:{}".format(item_name, short_hand), - username="{} username:{}".format(item_name, short_hand), - )[0] - return user - def dummy_contact(self, item_name, short_hand): """Creates a dummy contact object""" contact = Contact.objects.get_or_create( @@ -175,6 +166,16 @@ class AuditedAdminMockData: phone="(555) 555 5555", )[0] return contact + + def dummy_user(self, item_name, short_hand): + """Creates a dummy user object, + but with a shorthand and support for multiple""" + user = User.objects.get_or_create( + first_name="{} first_name:{}".format(item_name, short_hand), + last_name="{} last_name:{}".format(item_name, short_hand), + username="{} username:{}".format(item_name, str(uuid.uuid4())[:8]), + )[0] + return user def dummy_draft_domain(self, item_name, prebuilt=False): """ diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 526a9ea2a..82af3573c 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -1108,8 +1108,8 @@ class AuditedAdminTest(TestCase): tested_fields = [ DomainApplication.authorizing_official.field, DomainApplication.submitter.field, - DomainApplication.investigator.field, - DomainApplication.creator.field, + # DomainApplication.investigator.field, + # DomainApplication.creator.field, DomainApplication.requested_domain.field, ] @@ -1164,7 +1164,7 @@ class AuditedAdminTest(TestCase): tested_fields = [ DomainInformation.authorizing_official.field, DomainInformation.submitter.field, - DomainInformation.creator.field, + # DomainInformation.creator.field, (DomainInformation.domain.field, ["name"]), (DomainInformation.domain_application.field, ["requested_domain__name"]), ] From 3bd9a2fbaa845443b9a0555d3cd37f6a628e1e29 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 21:25:36 -0500 Subject: [PATCH 48/78] lint --- src/registrar/tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 347781ef8..4535084af 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -166,7 +166,7 @@ class AuditedAdminMockData: phone="(555) 555 5555", )[0] return contact - + def dummy_user(self, item_name, short_hand): """Creates a dummy user object, but with a shorthand and support for multiple""" From ed5d4fc4c3126dd2f5dabfdccaa9bdb7a56496b0 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 12 Dec 2023 21:28:14 -0500 Subject: [PATCH 49/78] cleanup --- src/registrar/tests/common.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 4535084af..87bad0dfb 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -156,6 +156,16 @@ class AuditedAdminMockData: APPLICATION = "application" INVITATION = "invitation" + def dummy_user(self, item_name, short_hand): + """Creates a dummy user object, + but with a shorthand and support for multiple""" + user = User.objects.get_or_create( + first_name="{} first_name:{}".format(item_name, short_hand), + last_name="{} last_name:{}".format(item_name, short_hand), + username="{} username:{}".format(item_name, str(uuid.uuid4())[:8]), + )[0] + return user + def dummy_contact(self, item_name, short_hand): """Creates a dummy contact object""" contact = Contact.objects.get_or_create( @@ -167,16 +177,6 @@ class AuditedAdminMockData: )[0] return contact - def dummy_user(self, item_name, short_hand): - """Creates a dummy user object, - but with a shorthand and support for multiple""" - user = User.objects.get_or_create( - first_name="{} first_name:{}".format(item_name, short_hand), - last_name="{} last_name:{}".format(item_name, short_hand), - username="{} username:{}".format(item_name, str(uuid.uuid4())[:8]), - )[0] - return user - def dummy_draft_domain(self, item_name, prebuilt=False): """ Creates a dummy DraftDomain object From 2497b00fa3b40369e910c400c11d457b877a9f27 Mon Sep 17 00:00:00 2001 From: Kristina Yin Date: Tue, 12 Dec 2023 19:58:23 -0800 Subject: [PATCH 50/78] missed part of Tribal AO ex description --- src/registrar/templates/includes/ao_example.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/registrar/templates/includes/ao_example.html b/src/registrar/templates/includes/ao_example.html index 976deb34d..38bbe20ff 100644 --- a/src/registrar/templates/includes/ao_example.html +++ b/src/registrar/templates/includes/ao_example.html @@ -57,5 +57,6 @@ {% elif organization_type == 'tribal' %}

Tribal governments

Domain requests from federally recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

+

Domain requests from state-recognized tribal governments must be authorized by the tribal leader the individual state recognizes.

{% endif %} From e70324443183807bd75f42cd62843468824a5c2c Mon Sep 17 00:00:00 2001 From: Michelle Rago <60157596+michelle-rago@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:26:37 -0500 Subject: [PATCH 51/78] Hyphenated "federally-recognized" --- src/registrar/templates/includes/ao_example.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/includes/ao_example.html b/src/registrar/templates/includes/ao_example.html index 38bbe20ff..be5f4624d 100644 --- a/src/registrar/templates/includes/ao_example.html +++ b/src/registrar/templates/includes/ao_example.html @@ -56,7 +56,7 @@ {% elif organization_type == 'tribal' %}

Tribal governments

-

Domain requests from federally recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

+

Domain requests from federally-recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

Domain requests from state-recognized tribal governments must be authorized by the tribal leader the individual state recognizes.

{% endif %} From 9cc018beeba27eaa7bf20feaf5f9027a7ebcce6f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:18:37 -0700 Subject: [PATCH 52/78] Remove valid range check --- .../management/commands/extend_expiration_dates.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index a9078b275..57ea7b3e2 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -90,6 +90,7 @@ class Command(BaseCommand): self.update_skipped.append(domain.name) logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") else: + logger.info("What is the amount? {}") domain.renew_domain(extension_amount) self.update_success.append(domain.name) logger.info( @@ -120,13 +121,8 @@ class Command(BaseCommand): transition_domains = TransitionDomain.objects.filter( domain_name=domain.name, epp_expiration_date=current_expiration_date ) - proposed_date = self.add_years(current_expiration_date, extension_amount) - minimum_extension_date = self.add_years(self.expiration_cutoff, extension_amount) - maximum_extension_date = self.add_years(date(2025, 12, 31), extension_amount) - valid_range = minimum_extension_date <= proposed_date <= maximum_extension_date - - return valid_range and transition_domains.count() > 0 + return transition_domains.count() > 0 def prompt_user_to_proceed(self, extension_amount, domains_to_change_count): """Asks if the user wants to proceed with this action""" From e350d1e9e332845f630566cf0567f9caeb5ea7c4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:22:53 -0700 Subject: [PATCH 53/78] Update extend_expiration_dates.py --- .../commands/extend_expiration_dates.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 57ea7b3e2..f969faa62 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -90,7 +90,6 @@ class Command(BaseCommand): self.update_skipped.append(domain.name) logger.info(f"{TerminalColors.YELLOW}" f"Skipping update for {domain}" f"{TerminalColors.ENDC}") else: - logger.info("What is the amount? {}") domain.renew_domain(extension_amount) self.update_success.append(domain.name) logger.info( @@ -208,18 +207,3 @@ class Command(BaseCommand): {TerminalColors.ENDC} """ ) - - # We use this manual approach rather than relative delta due to our - # github localenv not having the package installed. - # Credit: https://stackoverflow.com/questions/15741618/add-one-year-in-current-date-python - def add_years(self, old_date, years): - """Return a date that's `years` years after the date (or datetime) - object `old_date`. Return the same calendar date (month and day) in the - destination year, if it exists, otherwise use the following day - (thus changing February 29 to March 1). - - """ - try: - return old_date.replace(year=old_date.year + years) - except ValueError: - return old_date + (date(old_date.year + years, 1, 1) - date(old_date.year, 1, 1)) From d89ace47ba2f7c35c371502620f4cdd8403d0a38 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:53:12 -0700 Subject: [PATCH 54/78] Fix migration weirdness --- ...ter_domain_state_alter_domainapplication_status_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/registrar/migrations/{0055_alter_domain_state_alter_domainapplication_status_and_more.py => 0056_alter_domain_state_alter_domainapplication_status_and_more.py} (96%) diff --git a/src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py b/src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py similarity index 96% rename from src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py rename to src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py index 9b6bac48c..097cddf8a 100644 --- a/src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py +++ b/src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py @@ -6,7 +6,7 @@ import django_fsm class Migration(migrations.Migration): dependencies = [ - ("registrar", "0054_alter_domainapplication_federal_agency_and_more"), + ("registrar", "0055_transitiondomain_processed"), ] operations = [ From aa0b8cc4b368c5a3801e3d167abf621381dd0842 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:29:58 -0700 Subject: [PATCH 55/78] Add documentation --- docs/operations/data_migration.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 2ce34dc76..a45e27982 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -493,3 +493,34 @@ The `load_organization_data` script has five optional parameters. These are as f | 3 | **directory** | Specifies the directory containing the files that will be parsed. Defaults to "migrationdata" | | 4 | **domain_additional_filename** | Specifies the filename of domain_additional. Used as an override for the JSON. Has no default. | | 5 | **organization_adhoc_filename** | Specifies the filename of organization_adhoc. Used as an override for the JSON. Has no default. | + + +## Extend Domain Extension Dates +This section outlines how to extend the expiration date of all ready domains (or a select subset) by a defined period of time. + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-za` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Extend domains +```./manage.py extend_expiration_dates``` + +### Running locally +```docker-compose exec app ./manage.py extend_expiration_dates``` + +##### Optional parameters +| | Parameter | Description | +|:-:|:-------------------------- |:----------------------------------------------------------------------------| +| 1 | **extensionAmount** | Determines the period of time to extend by, in years. Defaults to 1 year. | +| 2 | **debug** | Increases logging detail. Defaults to False. | +| 3 | **limitParse** | Determines how many domains to parse. Defaults to all. | +| 4 | **disableIdempotentCheck** | Boolean that determines if we should check for idempotence or not. Compares the proposed extension date to the value in TransitionDomains. Defaults to False. | From d2242c7667720ce5108d88c3536ecde11dd2e66d Mon Sep 17 00:00:00 2001 From: Kristina Yin Date: Wed, 13 Dec 2023 10:04:05 -0800 Subject: [PATCH 56/78] made sure external link icon shows up --- src/registrar/templates/includes/ao_example.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/templates/includes/ao_example.html b/src/registrar/templates/includes/ao_example.html index be5f4624d..85df174aa 100644 --- a/src/registrar/templates/includes/ao_example.html +++ b/src/registrar/templates/includes/ao_example.html @@ -2,7 +2,7 @@ {% if federal_type == 'executive' %}

Executive branch federal agencies

Domain requests from executive branch federal agencies must be authorized by the agency's CIO or the head of the agency.

-

See OMB Memorandum M-23-10 for more information.

+

See OMB Memorandum M-23-10 for more information.

{% elif federal_type == 'judicial' %}

Judicial branch federal agencies

@@ -56,7 +56,7 @@ {% elif organization_type == 'tribal' %}

Tribal governments

-

Domain requests from federally-recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

+

Domain requests from federally-recognized tribal governments must be authorized by the tribal leader the Bureau of Indian Affairs recognizes.

Domain requests from state-recognized tribal governments must be authorized by the tribal leader the individual state recognizes.

{% endif %} From 4adeb6722b9611b294c5ccec8a3edcdce15f00a0 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 13 Dec 2023 15:13:04 -0500 Subject: [PATCH 57/78] Revisions and clean up based on PR comments --- src/djangooidc/backends.py | 5 ++-- .../copy_names_from_contacts_to_users.py | 28 ++++++------------- src/registrar/tests/common.py | 2 +- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/djangooidc/backends.py b/src/djangooidc/backends.py index cf326eca4..7a192e4a5 100644 --- a/src/djangooidc/backends.py +++ b/src/djangooidc/backends.py @@ -65,8 +65,9 @@ class OpenIdConnectBackend(ModelBackend): return user def update_existing_user(self, user, kwargs): - # Update other fields without overwriting first_name and last_name. - # Overwrite first_name and last_name if not empty string + """Update other fields without overwriting first_name and last_name. + Overwrite first_name and last_name if not empty string""" + for key, value in kwargs.items(): # Check if the key is not first_name or last_name or value is not empty string if key not in ["first_name", "last_name"] or value: diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index c5e2364e9..6d089f721 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): help = """Copy first and last names from a contact to a related user if it exists and if its first and last name - properties are null""" + properties are null or blank strings.""" # ====================================================== # ===================== ARGUMENTS ===================== @@ -112,12 +112,12 @@ class Command(BaseCommand): # ---- UPDATE THE USER IF IT DOES NOT HAVE A FIRST AND LAST NAMES # ---- LET'S KEEP A LIGHT TOUCH if not eligible_user.first_name and not eligible_user.last_name: - processed_user = eligible_user # (expression has type "str | None", variable has type "str | int | Combinable") # so we'll ignore type - processed_user.first_name = contact.first_name # type: ignore - processed_user.last_name = contact.last_name # type: ignore - processed_user.save() + eligible_user.first_name = contact.first_name # type: ignore + eligible_user.last_name = contact.last_name # type: ignore + eligible_user.save() + processed_user = eligible_user return ( eligible_user, @@ -147,9 +147,9 @@ class Command(BaseCommand): def process_contacts( self, debug_on, - skipped_contacts, - eligible_users, - processed_users, + skipped_contacts=[], + eligible_users=[], + processed_users=[], ): for contact in Contact.objects.all(): # DEBUG: @@ -207,15 +207,6 @@ class Command(BaseCommand): self.print_debug_mode_statements(debug_on) - # users we SKIPPED - skipped_contacts = [] - - # users we found that are linked to contacts - eligible_users = [] - - # users we PROCESSED - processed_users = [] - logger.info( f"""{TerminalColors.OKCYAN} ========================== @@ -235,9 +226,6 @@ class Command(BaseCommand): processed_users, ) = self.process_contacts( debug_on, - skipped_contacts, - eligible_users, - processed_users, ) self.print_summary_of_findings( diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 87bad0dfb..443c05ee3 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -162,7 +162,7 @@ class AuditedAdminMockData: user = User.objects.get_or_create( first_name="{} first_name:{}".format(item_name, short_hand), last_name="{} last_name:{}".format(item_name, short_hand), - username="{} username:{}".format(item_name, str(uuid.uuid4())[:8]), + username="{} username:{}".format(item_name + str(uuid.uuid4())[:8], short_hand), )[0] return user From 9c8ed1682bcc78e3f239b765b88b20020878f140 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 13 Dec 2023 15:41:18 -0500 Subject: [PATCH 58/78] linter --- src/djangooidc/backends.py | 2 +- src/registrar/tests/common.py | 4 ++-- src/registrar/tests/test_admin.py | 4 ++-- src/registrar/tests/test_models.py | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/djangooidc/backends.py b/src/djangooidc/backends.py index 7a192e4a5..b0e6417db 100644 --- a/src/djangooidc/backends.py +++ b/src/djangooidc/backends.py @@ -67,7 +67,7 @@ class OpenIdConnectBackend(ModelBackend): def update_existing_user(self, user, kwargs): """Update other fields without overwriting first_name and last_name. Overwrite first_name and last_name if not empty string""" - + for key, value in kwargs.items(): # Check if the key is not first_name or last_name or value is not empty string if key not in ["first_name", "last_name"] or value: diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 443c05ee3..ec09b45d0 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -406,8 +406,8 @@ def mock_user(): """A simple user.""" user_kwargs = dict( id=4, - first_name="Rachid", - last_name="Mrad", + first_name="Jeff", + last_name="Lebowski", ) mock_user, _ = User.objects.get_or_create(**user_kwargs) return mock_user diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 82af3573c..8cca67d1e 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -1021,7 +1021,7 @@ class ListHeaderAdminTest(TestCase): # Set the GET parameters for testing request.GET = { "status": "started", - "investigator": "Rachid Mrad", + "investigator": "Jeff Lebowski", "q": "search_value", } # Call the get_filters method @@ -1032,7 +1032,7 @@ class ListHeaderAdminTest(TestCase): filters, [ {"parameter_name": "status", "parameter_value": "started"}, - {"parameter_name": "investigator", "parameter_value": "Rachid Mrad"}, + {"parameter_name": "investigator", "parameter_value": "Jeff Lebowski"}, ], ) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index dfab4f9a3..d9090c287 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -659,7 +659,7 @@ class TestUser(TestCase): class TestContact(TestCase): def setUp(self): self.email = "mayor@igorville.gov" - self.user, _ = User.objects.get_or_create(email=self.email, first_name="Rachid", last_name="Mrad") + self.user, _ = User.objects.get_or_create(email=self.email, first_name="Jeff", last_name="Lebowski") self.contact, _ = Contact.objects.get_or_create(user=self.user) def tearDown(self): @@ -670,10 +670,10 @@ class TestContact(TestCase): def test_saving_contact_updates_user_first_last_names(self): """When a contact is updated, we propagate the changes to the linked user if it exists.""" # User and Contact are created and linked as expected - self.assertEqual(self.contact.first_name, "Rachid") - self.assertEqual(self.contact.last_name, "Mrad") - self.assertEqual(self.user.first_name, "Rachid") - self.assertEqual(self.user.last_name, "Mrad") + self.assertEqual(self.contact.first_name, "Jeff") + self.assertEqual(self.contact.last_name, "Lebowski") + self.assertEqual(self.user.first_name, "Jeff") + self.assertEqual(self.user.last_name, "Lebowski") self.contact.first_name = "Joey" self.contact.last_name = "Baloney" From 52220d0a9e8cc9533d0ad79b66827ee30e07d2f1 Mon Sep 17 00:00:00 2001 From: Michelle Rago <60157596+michelle-rago@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:13:33 -0500 Subject: [PATCH 59/78] Changed DNS sidebar nav / Put DNS options in a list on DNS landing page (#1389) * Changed domain management sidebar nav for DNS items * Bulleted DNS options on DNS landing page * Remove "DNS" from "DNS name servers" * Remove "DNS" from "DNS name servers" --- src/registrar/templates/domain_dns.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/registrar/templates/domain_dns.html b/src/registrar/templates/domain_dns.html index 85ba02e16..0f625e0e3 100644 --- a/src/registrar/templates/domain_dns.html +++ b/src/registrar/templates/domain_dns.html @@ -12,9 +12,11 @@

You can enter your name servers, as well as other DNS-related information, in the following sections:

{% url 'domain-dns-nameservers' pk=domain.id as url %} -

DNS name servers

+ {% endblock %} {# domain_content #} From e36351d21ef5d60be8a4d96579b4dfd9c446bd71 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 14 Dec 2023 07:44:31 -0500 Subject: [PATCH 60/78] lowercase emails on add user to domain; case insensitive match on matching DomainInvitations on login; test cases --- src/registrar/forms/domain.py | 11 +++++++++++ src/registrar/models/user.py | 2 +- src/registrar/tests/test_models.py | 14 ++++++++++++++ src/registrar/tests/test_views.py | 26 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 8b55aa29d..ac96393b4 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -28,6 +28,17 @@ class DomainAddUserForm(forms.Form): email = forms.EmailField(label="Email") + def clean(self): + """clean form data by lowercasing email""" + cleaned_data = super().clean() + + # Lowercase the value of the 'email' field + email_value = cleaned_data.get("email") + if email_value: + cleaned_data["email"] = email_value.lower() + + return cleaned_data + class DomainNameserverForm(forms.Form): """Form for changing nameservers.""" diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index ec2d06c70..d79e4c9ee 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -101,7 +101,7 @@ class User(AbstractUser): """When a user first arrives on the site, we need to retrieve any domain invitations that match their email address.""" for invitation in DomainInvitation.objects.filter( - email=self.email, status=DomainInvitation.DomainInvitationStatus.INVITED + email__iexact=self.email, status=DomainInvitation.DomainInvitationStatus.INVITED ): try: invitation.retrieve() diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 83126ab7c..ca221a18f 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -654,3 +654,17 @@ class TestUser(TestCase): """A new user who's neither transitioned nor invited should return True when tested with class method needs_identity_verification""" self.assertTrue(User.needs_identity_verification(self.user.email, self.user.username)) + + def test_check_domain_invitations_on_login_caps_email(self): + """A DomainInvitation with an email address with capital letters should match + a User record whose email address is not in caps""" + # create DomainInvitation with CAPS email that matches User email + # on a case-insensitive match + CAPS_EMAIL = "MAYOR@igorville.gov" + # mock the domain invitation save routine + with patch("registrar.models.DomainInvitation.save") as save_mock: + DomainInvitation.objects.get_or_create(email=CAPS_EMAIL, domain=self.domain) + self.user.check_domain_invitations_on_login() + # if check_domain_invitations_on_login properly matches exactly one + # Domain Invitation, then save routine should be called exactly once + save_mock.assert_called_once diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index da6fe6205..89ef0e35e 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1372,6 +1372,32 @@ class TestDomainManagers(TestDomainOverview): self.assertContains(success_page, "Cancel") # link to cancel invitation self.assertTrue(DomainInvitation.objects.filter(email=EMAIL).exists()) + @boto3_mocking.patching + def test_domain_invitation_created_for_caps_email(self): + """Add user on a nonexistent email with CAPS creates an invitation to lowercase email. + + Adding a non-existent user sends an email as a side-effect, so mock + out the boto3 SES email sending here. + """ + # make sure there is no user with this email + EMAIL = "mayor@igorville.gov" + CAPS_EMAIL = "MAYOR@igorville.gov" + User.objects.filter(email=EMAIL).delete() + + self.domain_information, _ = DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain) + + add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + add_page.form["email"] = CAPS_EMAIL + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + success_result = add_page.form.submit() + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + success_page = success_result.follow() + + self.assertContains(success_page, EMAIL) + self.assertContains(success_page, "Cancel") # link to cancel invitation + self.assertTrue(DomainInvitation.objects.filter(email=EMAIL).exists()) + @boto3_mocking.patching def test_domain_invitation_email_sent(self): """Inviting a non-existent user sends them an email.""" From 7b7653ac9822ea175d54292780f2360cb81bf928 Mon Sep 17 00:00:00 2001 From: Katherine-Osos <119689946+Katherine-Osos@users.noreply.github.com> Date: Thu, 14 Dec 2023 09:55:32 -0600 Subject: [PATCH 61/78] Wording updates --- src/registrar/templates/home.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/templates/home.html b/src/registrar/templates/home.html index 8e056924f..362b14f18 100644 --- a/src/registrar/templates/home.html +++ b/src/registrar/templates/home.html @@ -87,7 +87,7 @@ aria-live="polite" > {% else %} -

You don't have any registered domains yet

+

You don't have any registered domains.

{% endif %} @@ -95,7 +95,7 @@

Domain requests

{% if domain_applications %} - + @@ -138,7 +138,7 @@ aria-live="polite" > {% else %} -

You don't have any domain requests yet

+

You haven't requested any domains.

{% endif %} From 7e4f500f941f640b7b39193e55796b337d49f7e6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 14 Dec 2023 11:06:43 -0500 Subject: [PATCH 62/78] updated formatting and variable names in tests --- src/registrar/tests/test_models.py | 6 ++-- src/registrar/tests/test_views.py | 46 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index ca221a18f..8642b1d15 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -660,11 +660,11 @@ class TestUser(TestCase): a User record whose email address is not in caps""" # create DomainInvitation with CAPS email that matches User email # on a case-insensitive match - CAPS_EMAIL = "MAYOR@igorville.gov" + caps_email = "MAYOR@igorville.gov" # mock the domain invitation save routine with patch("registrar.models.DomainInvitation.save") as save_mock: - DomainInvitation.objects.get_or_create(email=CAPS_EMAIL, domain=self.domain) + DomainInvitation.objects.get_or_create(email=caps_email, domain=self.domain) self.user.check_domain_invitations_on_login() # if check_domain_invitations_on_login properly matches exactly one # Domain Invitation, then save routine should be called exactly once - save_mock.assert_called_once + save_mock.assert_called_once() diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 89ef0e35e..17641636e 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1355,22 +1355,22 @@ class TestDomainManagers(TestDomainOverview): out the boto3 SES email sending here. """ # make sure there is no user with this email - EMAIL = "mayor@igorville.gov" - User.objects.filter(email=EMAIL).delete() + email_address = "mayor@igorville.gov" + User.objects.filter(email=email_address).delete() self.domain_information, _ = DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain) add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - add_page.form["email"] = EMAIL + add_page.form["email"] = email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_result = add_page.form.submit() self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() - self.assertContains(success_page, EMAIL) + self.assertContains(success_page, email_address) self.assertContains(success_page, "Cancel") # link to cancel invitation - self.assertTrue(DomainInvitation.objects.filter(email=EMAIL).exists()) + self.assertTrue(DomainInvitation.objects.filter(email=email_address).exists()) @boto3_mocking.patching def test_domain_invitation_created_for_caps_email(self): @@ -1380,30 +1380,30 @@ class TestDomainManagers(TestDomainOverview): out the boto3 SES email sending here. """ # make sure there is no user with this email - EMAIL = "mayor@igorville.gov" - CAPS_EMAIL = "MAYOR@igorville.gov" - User.objects.filter(email=EMAIL).delete() + email_address = "mayor@igorville.gov" + caps_email_address = "MAYOR@igorville.gov" + User.objects.filter(email=email_address).delete() self.domain_information, _ = DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain) add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - add_page.form["email"] = CAPS_EMAIL + add_page.form["email"] = caps_email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_result = add_page.form.submit() self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() - self.assertContains(success_page, EMAIL) + self.assertContains(success_page, email_address) self.assertContains(success_page, "Cancel") # link to cancel invitation - self.assertTrue(DomainInvitation.objects.filter(email=EMAIL).exists()) + self.assertTrue(DomainInvitation.objects.filter(email=email_address).exists()) @boto3_mocking.patching def test_domain_invitation_email_sent(self): """Inviting a non-existent user sends them an email.""" # make sure there is no user with this email - EMAIL = "mayor@igorville.gov" - User.objects.filter(email=EMAIL).delete() + email_address = "mayor@igorville.gov" + User.objects.filter(email=email_address).delete() self.domain_information, _ = DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain) @@ -1412,28 +1412,28 @@ class TestDomainManagers(TestDomainOverview): with boto3_mocking.clients.handler_for("sesv2", mock_client): add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - add_page.form["email"] = EMAIL + add_page.form["email"] = email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) add_page.form.submit() # check the mock instance to see if `send_email` was called right mock_client_instance.send_email.assert_called_once_with( FromEmailAddress=settings.DEFAULT_FROM_EMAIL, - Destination={"ToAddresses": [EMAIL]}, + Destination={"ToAddresses": [email_address]}, Content=ANY, ) def test_domain_invitation_cancel(self): """Posting to the delete view deletes an invitation.""" - EMAIL = "mayor@igorville.gov" - invitation, _ = DomainInvitation.objects.get_or_create(domain=self.domain, email=EMAIL) + email_address = "mayor@igorville.gov" + invitation, _ = DomainInvitation.objects.get_or_create(domain=self.domain, email=email_address) self.client.post(reverse("invitation-delete", kwargs={"pk": invitation.id})) with self.assertRaises(DomainInvitation.DoesNotExist): DomainInvitation.objects.get(id=invitation.id) def test_domain_invitation_cancel_no_permissions(self): """Posting to the delete view as a different user should fail.""" - EMAIL = "mayor@igorville.gov" - invitation, _ = DomainInvitation.objects.get_or_create(domain=self.domain, email=EMAIL) + email_address = "mayor@igorville.gov" + invitation, _ = DomainInvitation.objects.get_or_create(domain=self.domain, email=email_address) other_user = User() other_user.save() @@ -1445,20 +1445,20 @@ class TestDomainManagers(TestDomainOverview): @boto3_mocking.patching def test_domain_invitation_flow(self): """Send an invitation to a new user, log in and load the dashboard.""" - EMAIL = "mayor@igorville.gov" - User.objects.filter(email=EMAIL).delete() + email_address = "mayor@igorville.gov" + User.objects.filter(email=email_address).delete() add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) self.domain_information, _ = DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - add_page.form["email"] = EMAIL + add_page.form["email"] = email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) add_page.form.submit() # user was invited, create them - new_user = User.objects.create(username=EMAIL, email=EMAIL) + new_user = User.objects.create(username=email_address, email=email_address) # log them in to `self.app` self.app.set_user(new_user.username) # and manually call the on each login callback From 4727ce6b63459f2990e89cd0a647b60a0a9bcbf3 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 14 Dec 2023 10:41:44 -0700 Subject: [PATCH 63/78] PR suggestion --- src/registrar/management/commands/extend_expiration_dates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index f969faa62..dd366cddd 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -134,7 +134,7 @@ class Command(BaseCommand): ==Proposed Changes== Domains to change: {domains_to_change_count} """, - prompt_title="Do you wish to proceed?", + prompt_title="Do you wish to proceed with these changes?", ) logger.info(f"{TerminalColors.MAGENTA}" "Preparing to extend expiration dates..." f"{TerminalColors.ENDC}") From 7bcf148453f204491bee8f3c8a170952e057e9ec Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Dec 2023 13:11:01 -0500 Subject: [PATCH 64/78] cleanup --- .../commands/copy_names_from_contacts_to_users.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py index 6d089f721..50e1bea3d 100644 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ b/src/registrar/management/commands/copy_names_from_contacts_to_users.py @@ -89,7 +89,8 @@ class Command(BaseCommand): """Given a contact with a first_name and last_name, find & update an existing corresponding user if her first_name and last_name are null. - Returns the corresponding User object. + Returns tuple of eligible (is linked to the contact) and processed + (first and last are blank) users. """ user_exists = User.objects.filter(contact=contact).exists() @@ -124,7 +125,7 @@ class Command(BaseCommand): processed_user, ) - except Exception as E: + except Exception as error: logger.warning( f""" {TerminalColors.FAIL} @@ -132,7 +133,7 @@ class Command(BaseCommand): User table for the following user: {contact.email} {contact.first_name} {contact.last_name} - Exception is: {E} + Exception is: {error} ----------TERMINATING----------""" ) sys.exit() @@ -143,7 +144,6 @@ class Command(BaseCommand): # ================= PROCESS CONTACTS ================== # ====================================================== - # C901 'Command.handle' is too complex def process_contacts( self, debug_on, @@ -152,7 +152,6 @@ class Command(BaseCommand): processed_users=[], ): for contact in Contact.objects.all(): - # DEBUG: TerminalHelper.print_conditional( debug_on, f"{TerminalColors.OKCYAN}" @@ -160,7 +159,7 @@ class Command(BaseCommand): f"{contact.email}," f" {contact.first_name}," f" {contact.last_name}" - f"{TerminalColors.ENDC}", # noqa + f"{TerminalColors.ENDC}", ) # ====================================================== From ea7d34907c0bff77464a9a5bc9730b0d7d862bcb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:23:11 -0700 Subject: [PATCH 65/78] Add max cutoff date --- .../management/commands/extend_expiration_dates.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index dd366cddd..84d26e544 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -29,7 +29,8 @@ class Command(BaseCommand): self.update_success = [] self.update_skipped = [] self.update_failed = [] - self.expiration_cutoff = date(2023, 11, 15) + self.expiration_minimum_cutoff = date(2023, 11, 15) + self.expiration_maximum_cutoff = date(2023, 12, 30) def add_arguments(self, parser): """Add command line arguments.""" @@ -71,7 +72,9 @@ class Command(BaseCommand): self.check_if_positive_int(limit_parse, "limitParse") valid_domains = Domain.objects.filter( - expiration_date__gte=self.expiration_cutoff, state=Domain.State.READY + expiration_date__gte=self.expiration_minimum_cutoff, + expiration_date__lte=self.expiration_maximum_cutoff, + state=Domain.State.READY ).order_by("name") domains_to_change_count = valid_domains.count() From 04d3b92f9665fc676e3cbdda8f7415de963e697b Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Dec 2023 13:23:27 -0500 Subject: [PATCH 66/78] fix merge error: misplaced test: test_check_domain_invitations_on_login_caps_email --- src/registrar/tests/test_models.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 3854528b4..51a89a359 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -654,6 +654,20 @@ class TestUser(TestCase): """A new user who's neither transitioned nor invited should return True when tested with class method needs_identity_verification""" self.assertTrue(User.needs_identity_verification(self.user.email, self.user.username)) + + def test_check_domain_invitations_on_login_caps_email(self): + """A DomainInvitation with an email address with capital letters should match + a User record whose email address is not in caps""" + # create DomainInvitation with CAPS email that matches User email + # on a case-insensitive match + caps_email = "MAYOR@igorville.gov" + # mock the domain invitation save routine + with patch("registrar.models.DomainInvitation.save") as save_mock: + DomainInvitation.objects.get_or_create(email=caps_email, domain=self.domain) + self.user.check_domain_invitations_on_login() + # if check_domain_invitations_on_login properly matches exactly one + # Domain Invitation, then save routine should be called exactly once + save_mock.assert_called_once() class TestContact(TestCase): @@ -700,16 +714,3 @@ class TestContact(TestCase): self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") self.assertEqual(self.user.email, "mayor@igorville.gov") - def test_check_domain_invitations_on_login_caps_email(self): - """A DomainInvitation with an email address with capital letters should match - a User record whose email address is not in caps""" - # create DomainInvitation with CAPS email that matches User email - # on a case-insensitive match - caps_email = "MAYOR@igorville.gov" - # mock the domain invitation save routine - with patch("registrar.models.DomainInvitation.save") as save_mock: - DomainInvitation.objects.get_or_create(email=caps_email, domain=self.domain) - self.user.check_domain_invitations_on_login() - # if check_domain_invitations_on_login properly matches exactly one - # Domain Invitation, then save routine should be called exactly once - save_mock.assert_called_once() From 2481e89c2a44b976cb64a1a58dc52357709d732e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:38:09 -0700 Subject: [PATCH 67/78] Add unit tests --- .../commands/extend_expiration_dates.py | 2 +- src/registrar/tests/common.py | 10 +++++++ .../test_transition_domain_migrations.py | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/extend_expiration_dates.py b/src/registrar/management/commands/extend_expiration_dates.py index 84d26e544..5e203e488 100644 --- a/src/registrar/management/commands/extend_expiration_dates.py +++ b/src/registrar/management/commands/extend_expiration_dates.py @@ -74,7 +74,7 @@ class Command(BaseCommand): valid_domains = Domain.objects.filter( expiration_date__gte=self.expiration_minimum_cutoff, expiration_date__lte=self.expiration_maximum_cutoff, - state=Domain.State.READY + state=Domain.State.READY, ).order_by("name") domains_to_change_count = valid_domains.count() diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 337b6f31e..ad34d53a7 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -835,6 +835,11 @@ class MockEppLib(TestCase): ex_date=datetime.date(2023, 2, 15), ) + mockMaximumRenewedDomainExpDate = fakedEppObject( + "fakemaximum.gov", + ex_date=datetime.date(2024, 12, 31), + ) + mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", ex_date=datetime.date(2024, 11, 15), @@ -948,6 +953,11 @@ class MockEppLib(TestCase): res_data=[self.mockDnsNeededRenewedDomainExpDate], code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, ) + elif getattr(_request, "name", None) == "fakemaximum.gov": + return MagicMock( + res_data=[self.mockMaximumRenewedDomainExpDate], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) else: return MagicMock( res_data=[self.mockRenewedDomainExpDate], diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 2da42462c..363dafc2c 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -52,6 +52,15 @@ class TestExtendExpirationDates(MockEppLib): domain_name="fakeneeded.gov", epp_expiration_date=datetime.date(2023, 11, 15), ) + # Create a domain with a date greater than the maximum + Domain.objects.get_or_create( + name="fakemaximum.gov", state=Domain.State.READY, expiration_date=datetime.date(2024, 12, 31) + ) + TransitionDomain.objects.get_or_create( + username="fakemaximum@mail.com", + domain_name="fakemaximum.gov", + epp_expiration_date=datetime.date(2024, 12, 31), + ) def tearDown(self): """Deletes all DB objects related to migrations""" @@ -114,6 +123,25 @@ class TestExtendExpirationDates(MockEppLib): # should not be affected by the change. self.assertEqual(current_domain.expiration_date, datetime.date(2022, 5, 25)) + def test_extends_expiration_date_skips_maximum_date(self): + """ + Tests that the extend_expiration_dates method correctly skips domains + with an expiration date more than a certain threshold. + """ + desired_domain = Domain.objects.filter(name="fakemaximum.gov").get() + desired_domain.expiration_date = datetime.date(2024, 12, 31) + + # Run the expiration date script + self.run_extend_expiration_dates() + + current_domain = Domain.objects.filter(name="fakemaximum.gov").get() + self.assertEqual(desired_domain, current_domain) + + # Explicitly test the expiration date. The extend_expiration_dates script + # will skip all dates less than date(2023, 11, 15), meaning that this domain + # should not be affected by the change. + self.assertEqual(current_domain.expiration_date, datetime.date(2024, 12, 31)) + def test_extends_expiration_date_skips_non_ready(self): """ Tests that the extend_expiration_dates method correctly skips domains not in the state "ready" From c02f74cf6741e3b5149d8e9ced4b0e3c9a32ffaa Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Dec 2023 13:47:20 -0500 Subject: [PATCH 68/78] lint --- src/registrar/tests/test_models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 51a89a359..0e0839382 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -654,7 +654,7 @@ class TestUser(TestCase): """A new user who's neither transitioned nor invited should return True when tested with class method needs_identity_verification""" self.assertTrue(User.needs_identity_verification(self.user.email, self.user.username)) - + def test_check_domain_invitations_on_login_caps_email(self): """A DomainInvitation with an email address with capital letters should match a User record whose email address is not in caps""" @@ -713,4 +713,3 @@ class TestContact(TestCase): # Updating the contact's email does not propagate self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") self.assertEqual(self.user.email, "mayor@igorville.gov") - From d2e97c643292c413d54fd19b30efc0c0530dafb9 Mon Sep 17 00:00:00 2001 From: Katherine-Osos <119689946+Katherine-Osos@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:51:18 -0600 Subject: [PATCH 69/78] Org Type: Update definition for special district --- src/registrar/models/domain_application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index 12eda4caf..f6adffdd6 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -122,7 +122,7 @@ class DomainApplication(TimeStampedModel): FEDERAL = ( "federal", - "Federal: an agency of the U.S. government's executive, legislative, or judicial branches", + "Federal: an agency of the U.S. government’s executive, legislative, or judicial branches", ) INTERSTATE = "interstate", "Interstate: an organization of two or more states" STATE_OR_TERRITORY = ( @@ -139,7 +139,7 @@ class DomainApplication(TimeStampedModel): CITY = "city", "City: a city, town, township, village, etc." SPECIAL_DISTRICT = ( "special_district", - "Special district: an independent organization within a single state", + "Special district: an independent government that delivers specialized, essential services", ) SCHOOL_DISTRICT = ( "school_district", From 493f64e070eb43cc21c1516e076deab9c335287d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 14 Dec 2023 17:56:23 -0500 Subject: [PATCH 70/78] added code to json response for check availability so different message types can be handled differently by client --- src/api/views.py | 8 ++++---- src/registrar/utility/errors.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 068844919..0c24d0327 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -91,15 +91,15 @@ def available(request, domain=""): # validate that the given domain could be a domain name and fail early if # not. if not (DraftDomain.string_could_be_domain(domain) or DraftDomain.string_could_be_domain(domain + ".gov")): - return JsonResponse({"available": False, "message": DOMAIN_API_MESSAGES["invalid"]}) + return JsonResponse({"available": False, "code": "invalid", "message": DOMAIN_API_MESSAGES["invalid"]}) # a domain is available if it is NOT in the list of current domains try: if check_domain_available(domain): - return JsonResponse({"available": True, "message": DOMAIN_API_MESSAGES["success"]}) + return JsonResponse({"available": True, "code": "success", "message": DOMAIN_API_MESSAGES["success"]}) else: - return JsonResponse({"available": False, "message": DOMAIN_API_MESSAGES["unavailable"]}) + return JsonResponse({"available": False, "code": "unavailable", "message": DOMAIN_API_MESSAGES["unavailable"]}) except Exception: - return JsonResponse({"available": False, "message": DOMAIN_API_MESSAGES["error"]}) + return JsonResponse({"available": False, "code": "error", "message": DOMAIN_API_MESSAGES["error"]}) @require_http_methods(["GET"]) diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 9463c1387..d096a30da 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -43,11 +43,11 @@ class GenericError(Exception): """ _error_mapping = { - GenericErrorCodes.CANNOT_CONTACT_REGISTRY: """ -We’re experiencing a system connection error. Please wait a few minutes -and try again. If you continue to receive this error after a few tries, -contact help@get.gov. - """, + GenericErrorCodes.CANNOT_CONTACT_REGISTRY: ( +"We’re experiencing a system connection error. Please wait a few minutes " +"and try again. If you continue to receive this error after a few tries, " +"contact help@get.gov." + ), GenericErrorCodes.GENERIC_ERROR: ("Value entered was wrong."), } From e1de3dab68f730b69ebf055d598f1756d368395c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 14 Dec 2023 18:08:35 -0500 Subject: [PATCH 71/78] linting --- src/api/views.py | 4 +++- src/registrar/utility/errors.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 0c24d0327..a7dd7600a 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -97,7 +97,9 @@ def available(request, domain=""): if check_domain_available(domain): return JsonResponse({"available": True, "code": "success", "message": DOMAIN_API_MESSAGES["success"]}) else: - return JsonResponse({"available": False, "code": "unavailable", "message": DOMAIN_API_MESSAGES["unavailable"]}) + return JsonResponse( + {"available": False, "code": "unavailable", "message": DOMAIN_API_MESSAGES["unavailable"]} + ) except Exception: return JsonResponse({"available": False, "code": "error", "message": DOMAIN_API_MESSAGES["error"]}) diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index d096a30da..455419236 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -44,9 +44,9 @@ class GenericError(Exception): _error_mapping = { GenericErrorCodes.CANNOT_CONTACT_REGISTRY: ( -"We’re experiencing a system connection error. Please wait a few minutes " -"and try again. If you continue to receive this error after a few tries, " -"contact help@get.gov." + "We’re experiencing a system connection error. Please wait a few minutes " + "and try again. If you continue to receive this error after a few tries, " + "contact help@get.gov." ), GenericErrorCodes.GENERIC_ERROR: ("Value entered was wrong."), } From 53dd308d334620333dd4665ca79c449e291e4162 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 15 Dec 2023 01:30:46 -0500 Subject: [PATCH 72/78] Add modal between DA submit and form submit button, unit test that looks for modal trigger and modal data, fix DA unit tests that query and trigger the original form since we now have 2 forms on the page --- src/registrar/templates/application_form.html | 18 +++- .../test_copy_names_from_contacts_to_users.py | 4 +- src/registrar/tests/test_views.py | 98 ++++++++++--------- src/registrar/views/application.py | 7 ++ 4 files changed, 78 insertions(+), 49 deletions(-) diff --git a/src/registrar/templates/application_form.html b/src/registrar/templates/application_form.html index db72a1fc2..b57c807f6 100644 --- a/src/registrar/templates/application_form.html +++ b/src/registrar/templates/application_form.html @@ -85,16 +85,28 @@ class="usa-button usa-button--outline" >Save and return to manage your domains {% else %} - + aria-controls="toggle-submit-domain-request" + data-open-modal + >Submit your domain request {% endif %} {% endblock %} +
+ {% include 'includes/modal.html' with modal_heading="You are about to submit a domain request for yourcity.gov." modal_description="Once you submit this request, you won’t be able to make further edits until it’s reviewed by our staff. You’ll only be able to withdraw your request." modal_button=modal_button|safe %} +
+ {% block after_form_content %}{% endblock %} diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py index 2690578e0..988b57a4b 100644 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -18,8 +18,8 @@ class TestDataUpdates(TestCase): self.user2 = User.objects.create(username="user2") self.user3 = User.objects.create(username="user3") self.user4 = User.objects.create(username="user4") - # The last user created triggers the creation of a contact and attaches itself to it. @Neil wth is going on? - # This bs_user defuses that situation so we can test the code. + # The last user created triggers the creation of a contact and attaches itself to it. See signals. + # This bs_user defuses that situation (unwanted user-contact pairing) so we can test the code. self.bs_user = User.objects.create() self.contact1 = Contact.objects.create( diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 17641636e..fb875db9a 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -141,7 +141,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # 302 redirect to the first form page = self.app.get(reverse("application:")).follow() # submitting should get back the same page if the required field is empty - result = page.form.submit() + result = page.forms[0].submit() self.assertIn("What kind of U.S.-based government organization do you represent?", result) def test_application_multiple_applications_exist(self): @@ -178,11 +178,11 @@ class DomainApplicationTests(TestWithUser, WebTest): session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] # ---- TYPE PAGE ---- - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "federal" # test next button and validate data self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # should see results in db application = DomainApplication.objects.get() # there's only one self.assertEqual(application.organization_type, "federal") @@ -197,7 +197,7 @@ class DomainApplicationTests(TestWithUser, WebTest): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) federal_page = type_result.follow() - federal_form = federal_page.form + federal_form = federal_page.forms[0] federal_form["organization_federal-federal_type"] = "executive" # test next button @@ -216,7 +216,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) org_contact_page = federal_result.follow() - org_contact_form = org_contact_page.form + org_contact_form = org_contact_page.forms[0] # federal agency so we have to fill in federal_agency org_contact_form["organization_contact-federal_agency"] = "General Services Administration" org_contact_form["organization_contact-organization_name"] = "Testorg" @@ -249,7 +249,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) ao_page = org_contact_result.follow() - ao_form = ao_page.form + ao_form = ao_page.forms[0] ao_form["authorizing_official-first_name"] = "Testy ATO" ao_form["authorizing_official-last_name"] = "Tester ATO" ao_form["authorizing_official-title"] = "Chief Tester" @@ -276,7 +276,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) current_sites_page = ao_result.follow() - current_sites_form = current_sites_page.form + current_sites_form = current_sites_page.forms[0] current_sites_form["current_sites-0-website"] = "www.city.com" # test next button @@ -298,7 +298,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) dotgov_page = current_sites_result.follow() - dotgov_form = dotgov_page.form + dotgov_form = dotgov_page.forms[0] dotgov_form["dotgov_domain-requested_domain"] = "city" dotgov_form["dotgov_domain-0-alternative_domain"] = "city1" @@ -318,7 +318,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) purpose_page = dotgov_result.follow() - purpose_form = purpose_page.form + purpose_form = purpose_page.forms[0] purpose_form["purpose-purpose"] = "For all kinds of things." # test next button @@ -337,7 +337,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) your_contact_page = purpose_result.follow() - your_contact_form = your_contact_page.form + your_contact_form = your_contact_page.forms[0] your_contact_form["your_contact-first_name"] = "Testy you" your_contact_form["your_contact-last_name"] = "Tester you" @@ -365,7 +365,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) other_contacts_page = your_contact_result.follow() - other_contacts_form = other_contacts_page.form + other_contacts_form = other_contacts_page.forms[0] other_contacts_form["other_contacts-0-first_name"] = "Testy2" other_contacts_form["other_contacts-0-last_name"] = "Tester2" @@ -398,7 +398,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) anything_else_page = other_contacts_result.follow() - anything_else_form = anything_else_page.form + anything_else_form = anything_else_page.forms[0] anything_else_form["anything_else-anything_else"] = "Nothing else." @@ -418,7 +418,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) requirements_page = anything_else_result.follow() - requirements_form = requirements_page.form + requirements_form = requirements_page.forms[0] requirements_form["requirements-is_policy_acknowledged"] = True @@ -438,7 +438,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) review_page = requirements_result.follow() - review_form = review_page.form + review_form = review_page.forms[0] # Review page contains all the previously entered data # Let's make sure the long org name is displayed @@ -540,7 +540,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # the conditional step titles shouldn't appear initially self.assertNotContains(type_page, self.TITLES["organization_federal"]) self.assertNotContains(type_page, self.TITLES["organization_election"]) - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "federal" # set the session ID before .submit() @@ -561,9 +561,9 @@ class DomainApplicationTests(TestWithUser, WebTest): # continuing on in the flow we need to see top-level agency on the # contact page - federal_page.form["organization_federal-federal_type"] = "executive" + federal_page.forms[0]["organization_federal-federal_type"] = "executive" self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - federal_result = federal_page.form.submit() + federal_result = federal_page.forms[0].submit() # the post request should return a redirect to the contact # question self.assertEqual(federal_result.status_code, 302) @@ -586,7 +586,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # the conditional step titles shouldn't appear initially self.assertNotContains(type_page, self.TITLES["organization_federal"]) self.assertNotContains(type_page, self.TITLES["organization_election"]) - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "county" # set the session ID before .submit() @@ -606,9 +606,9 @@ class DomainApplicationTests(TestWithUser, WebTest): # continuing on in the flow we need to NOT see top-level agency on the # contact page - election_page.form["organization_election-is_election_board"] = "True" + election_page.forms[0]["organization_election-is_election_board"] = "True" self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - election_result = election_page.form.submit() + election_result = election_page.forms[0].submit() # the post request should return a redirect to the contact # question self.assertEqual(election_result.status_code, 302) @@ -626,10 +626,10 @@ class DomainApplicationTests(TestWithUser, WebTest): # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "federal" self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # follow first redirect self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) @@ -654,15 +654,15 @@ class DomainApplicationTests(TestWithUser, WebTest): # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = DomainApplication.OrganizationChoices.INTERSTATE self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # follow first redirect self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) contact_page = type_result.follow() - org_contact_form = contact_page.form + org_contact_form = contact_page.forms[0] self.assertNotIn("federal_agency", org_contact_form.fields) @@ -690,10 +690,10 @@ class DomainApplicationTests(TestWithUser, WebTest): # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = DomainApplication.OrganizationChoices.SPECIAL_DISTRICT self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_page.forms[0].submit() # follow first redirect self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) contact_page = type_result.follow() @@ -710,7 +710,7 @@ class DomainApplicationTests(TestWithUser, WebTest): session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - result = contacts_page.form.submit() + result = contacts_page.forms[0].submit() # follow first redirect self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) no_contacts_page = result.follow() @@ -727,10 +727,10 @@ class DomainApplicationTests(TestWithUser, WebTest): # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = DomainApplication.OrganizationChoices.INTERSTATE self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # follow first redirect self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) contact_page = type_result.follow() @@ -745,10 +745,10 @@ class DomainApplicationTests(TestWithUser, WebTest): # of a "session". We are going to do it manually, saving the session ID here # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = DomainApplication.OrganizationChoices.TRIBAL self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # the tribal government page comes immediately afterwards self.assertIn("/tribal_government", type_result.headers["Location"]) # follow first redirect @@ -767,18 +767,18 @@ class DomainApplicationTests(TestWithUser, WebTest): session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] # ---- TYPE PAGE ---- - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "federal" # test next button self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # ---- FEDERAL BRANCH PAGE ---- # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) federal_page = type_result.follow() - federal_form = federal_page.form + federal_form = federal_page.forms[0] federal_form["organization_federal-federal_type"] = "executive" self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) federal_result = federal_form.submit() @@ -787,7 +787,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) org_contact_page = federal_result.follow() - org_contact_form = org_contact_page.form + org_contact_form = org_contact_page.forms[0] # federal agency so we have to fill in federal_agency org_contact_form["organization_contact-federal_agency"] = "General Services Administration" org_contact_form["organization_contact-organization_name"] = "Testorg" @@ -828,18 +828,18 @@ class DomainApplicationTests(TestWithUser, WebTest): # and then setting the cookie on each request. session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] # ---- TYPE PAGE ---- - type_form = type_page.form + type_form = type_page.forms[0] type_form["organization_type-organization_type"] = "federal" # test next button self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - type_result = type_page.form.submit() + type_result = type_form.submit() # ---- FEDERAL BRANCH PAGE ---- # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) federal_page = type_result.follow() - federal_form = federal_page.form + federal_form = federal_page.forms[0] federal_form["organization_federal-federal_type"] = "executive" self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) federal_result = federal_form.submit() @@ -848,7 +848,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) org_contact_page = federal_result.follow() - org_contact_form = org_contact_page.form + org_contact_form = org_contact_page.forms[0] # federal agency so we have to fill in federal_agency org_contact_form["organization_contact-federal_agency"] = "General Services Administration" org_contact_form["organization_contact-organization_name"] = "Testorg" @@ -870,7 +870,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) ao_page = org_contact_result.follow() - ao_form = ao_page.form + ao_form = ao_page.forms[0] ao_form["authorizing_official-first_name"] = "Testy ATO" ao_form["authorizing_official-last_name"] = "Tester ATO" ao_form["authorizing_official-title"] = "Chief Tester" @@ -884,7 +884,7 @@ class DomainApplicationTests(TestWithUser, WebTest): # Follow the redirect to the next form page self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) current_sites_page = ao_result.follow() - current_sites_form = current_sites_page.form + current_sites_form = current_sites_page.forms[0] current_sites_form["current_sites-0-website"] = "www.city.com" # test saving the page @@ -917,7 +917,7 @@ class DomainApplicationTests(TestWithUser, WebTest): current_sites_page = self.app.get(reverse("application:current_sites")) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] # fill in the form field - current_sites_form = current_sites_page.form + current_sites_form = current_sites_page.forms[0] self.assertIn("current_sites-0-website", current_sites_form.fields) self.assertNotIn("current_sites-1-website", current_sites_form.fields) current_sites_form["current_sites-0-website"] = "https://example.com" @@ -926,7 +926,7 @@ class DomainApplicationTests(TestWithUser, WebTest): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) current_sites_result = current_sites_form.submit("submit_button", value="save") self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - current_sites_form = current_sites_result.follow().form + current_sites_form = current_sites_result.follow().forms[0] # verify that there are two form fields value = current_sites_form["current_sites-0-website"].value @@ -1086,6 +1086,16 @@ class DomainApplicationTests(TestWithUser, WebTest): detail_page = home_page.click("Manage", index=0) self.assertContains(detail_page, "Federal: an agency of the U.S. government") + def test_submit_modal(self): + """When user clicks on submit your domain request, a modal pops up.""" + completed_application() + review_page = self.app.get(reverse("application:review")) + # We can't test the modal itself as it relies on JS for init and triggering, + # but we can test for the existence of its trigger: + self.assertContains(review_page, "toggle-submit-domain-request") + # And the existence of the modal's data parked and ready for the js init: + self.assertContains(review_page, "You are about to submit a domain request") + class TestWithDomainPermissions(TestWithUser): def setUp(self): diff --git a/src/registrar/views/application.py b/src/registrar/views/application.py index bb1b3aee6..3bf12ec63 100644 --- a/src/registrar/views/application.py +++ b/src/registrar/views/application.py @@ -321,12 +321,19 @@ class ApplicationWizard(ApplicationWizardPermissionView, TemplateView): def get_context_data(self): """Define context for access on all wizard pages.""" + # Create HTML for the submit button: + # The on-page submit button is just a trigger for the modal; + # the submit button we're adding to context will get passed to + # the modal and is the button that triggers the actual domain + # application submission (via post -> goto_next_step -> done). + modal_button = '" return { "form_titles": self.TITLES, "steps": self.steps, # Add information about which steps should be unlocked "visited": self.storage.get("step_history", []), "is_federal": self.application.is_federal(), + "modal_button": modal_button, } def get_step_list(self) -> list: From 8b6ca02de1b52cdb3bdf77e1e49a5e4705c92c1c Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 15 Dec 2023 01:34:48 -0500 Subject: [PATCH 73/78] Make the modal a forced action modal --- src/registrar/templates/application_form.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/registrar/templates/application_form.html b/src/registrar/templates/application_form.html index b57c807f6..98eab0152 100644 --- a/src/registrar/templates/application_form.html +++ b/src/registrar/templates/application_form.html @@ -103,6 +103,7 @@ id="toggle-submit-domain-request" aria-labelledby="Are you sure you want to continue?" aria-describedby="Your DNSSEC records will be deleted from the registry." + data-force-action > {% include 'includes/modal.html' with modal_heading="You are about to submit a domain request for yourcity.gov." modal_description="Once you submit this request, you won’t be able to make further edits until it’s reviewed by our staff. You’ll only be able to withdraw your request." modal_button=modal_button|safe %} From 6912b03cc0a69a6d0ae426a88003cce1383ca76c Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 15 Dec 2023 02:36:04 -0500 Subject: [PATCH 74/78] Remove review page from pa11y, move the building of modal header into the view, edit test (wip) --- src/.pa11yci | 1 - src/registrar/templates/application_form.html | 6 +++--- src/registrar/tests/test_views.py | 5 +++-- src/registrar/views/application.py | 10 +++++++++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/.pa11yci b/src/.pa11yci index 6bb5727e0..0ab3f4dd7 100644 --- a/src/.pa11yci +++ b/src/.pa11yci @@ -19,7 +19,6 @@ "http://localhost:8080/register/other_contacts/", "http://localhost:8080/register/anything_else/", "http://localhost:8080/register/requirements/", - "http://localhost:8080/register/review/", "http://localhost:8080/register/finished/" ] } diff --git a/src/registrar/templates/application_form.html b/src/registrar/templates/application_form.html index 98eab0152..cec2416fb 100644 --- a/src/registrar/templates/application_form.html +++ b/src/registrar/templates/application_form.html @@ -101,11 +101,11 @@
- {% include 'includes/modal.html' with modal_heading="You are about to submit a domain request for yourcity.gov." modal_description="Once you submit this request, you won’t be able to make further edits until it’s reviewed by our staff. You’ll only be able to withdraw your request." modal_button=modal_button|safe %} + {% include 'includes/modal.html' with modal_heading=modal_heading|safe modal_description="Once you submit this request, you won’t be able to make further edits until it’s reviewed by our staff. You’ll only be able to withdraw your request." modal_button=modal_button|safe %}
{% block after_form_content %}{% endblock %} diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index fb875db9a..b5574c8e2 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1088,13 +1088,14 @@ class DomainApplicationTests(TestWithUser, WebTest): def test_submit_modal(self): """When user clicks on submit your domain request, a modal pops up.""" - completed_application() + # completed_application(name="cats.gov") review_page = self.app.get(reverse("application:review")) # We can't test the modal itself as it relies on JS for init and triggering, # but we can test for the existence of its trigger: self.assertContains(review_page, "toggle-submit-domain-request") # And the existence of the modal's data parked and ready for the js init: - self.assertContains(review_page, "You are about to submit a domain request") + print(review_page) + self.assertContains(review_page, "You are about to submit") class TestWithDomainPermissions(TestWithUser): diff --git a/src/registrar/views/application.py b/src/registrar/views/application.py index 3bf12ec63..4e76f55fe 100644 --- a/src/registrar/views/application.py +++ b/src/registrar/views/application.py @@ -321,12 +321,19 @@ class ApplicationWizard(ApplicationWizardPermissionView, TemplateView): def get_context_data(self): """Define context for access on all wizard pages.""" - # Create HTML for the submit button: # The on-page submit button is just a trigger for the modal; # the submit button we're adding to context will get passed to # the modal and is the button that triggers the actual domain # application submission (via post -> goto_next_step -> done). modal_button = '" + # We'll concatenate the modal header here for passing along to the + # modal include. NOTE: We are able to 'fast-forward' a domain application + # by tyoing in review in the URL. The submit button still shows, hence + # the if/else. + if self.application.requested_domain: + modal_heading = 'You are about to submit a domain request for ' + str(self.application.requested_domain) + else: + modal_heading = 'You are about to submit an incomplete request' return { "form_titles": self.TITLES, "steps": self.steps, @@ -334,6 +341,7 @@ class ApplicationWizard(ApplicationWizardPermissionView, TemplateView): "visited": self.storage.get("step_history", []), "is_federal": self.application.is_federal(), "modal_button": modal_button, + "modal_heading": modal_heading, } def get_step_list(self) -> list: From 41ca7390d61e35a0cd297cb3f8d6d1590c2ea89b Mon Sep 17 00:00:00 2001 From: Katherine-Osos <119689946+Katherine-Osos@users.noreply.github.com> Date: Fri, 15 Dec 2023 10:14:01 -0600 Subject: [PATCH 75/78] Update order of fed branches --- src/registrar/models/domain_application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index f6adffdd6..303f0cd22 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -122,7 +122,7 @@ class DomainApplication(TimeStampedModel): FEDERAL = ( "federal", - "Federal: an agency of the U.S. government’s executive, legislative, or judicial branches", + "Federal: an agency of the U.S. government’s legislative, executive, or judicial branches", ) INTERSTATE = "interstate", "Interstate: an organization of two or more states" STATE_OR_TERRITORY = ( From 8718635b94e6b044b9da60d3cc7bdd13f4302b34 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 15 Dec 2023 11:36:32 -0500 Subject: [PATCH 76/78] Update unit tests to look for the dynamic header passed in context to the modal --- src/registrar/tests/test_views.py | 28 ++++++++++++++++++++-------- src/registrar/views/application.py | 4 ++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index b5574c8e2..57fa03f52 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -164,6 +164,9 @@ class DomainApplicationTests(TestWithUser, WebTest): this test work. This test also looks for the long organization name on the summary page. + + This also tests for the presence of a modal trigger and the dynamic test + in the modal header on the submit page. """ num_pages_tested = 0 # elections, type_of_work, tribal_government, no_other_contacts @@ -472,6 +475,14 @@ class DomainApplicationTests(TestWithUser, WebTest): self.assertContains(review_page, "(201) 555-5557") self.assertContains(review_page, "Nothing else.") + # We can't test the modal itself as it relies on JS for init and triggering, + # but we can test for the existence of its trigger: + self.assertContains(review_page, "toggle-submit-domain-request") + # And the existence of the modal's data parked and ready for the js init. + # The next assert also tests for the passed requested domain context from + # the view > application_form > modal + self.assertContains(review_page, "You are about to submit a domain request for city.gov") + # final submission results in a redirect to the "finished" URL self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) with less_console_noise(): @@ -1086,16 +1097,17 @@ class DomainApplicationTests(TestWithUser, WebTest): detail_page = home_page.click("Manage", index=0) self.assertContains(detail_page, "Federal: an agency of the U.S. government") - def test_submit_modal(self): - """When user clicks on submit your domain request, a modal pops up.""" - # completed_application(name="cats.gov") + def test_submit_modal_no_domain_text_fallback(self): + """When user clicks on submit your domain request and the requested domain + is null (possible through url direct access to the review page), present + fallback copy in the modal's header. + + NOTE: This may be a moot point if we implement a more solid pattern in the + future, like not a submit action at all on the review page.""" + review_page = self.app.get(reverse("application:review")) - # We can't test the modal itself as it relies on JS for init and triggering, - # but we can test for the existence of its trigger: self.assertContains(review_page, "toggle-submit-domain-request") - # And the existence of the modal's data parked and ready for the js init: - print(review_page) - self.assertContains(review_page, "You are about to submit") + self.assertContains(review_page, "You are about to submit an incomplete request") class TestWithDomainPermissions(TestWithUser): diff --git a/src/registrar/views/application.py b/src/registrar/views/application.py index 4e76f55fe..ba716c117 100644 --- a/src/registrar/views/application.py +++ b/src/registrar/views/application.py @@ -331,9 +331,9 @@ class ApplicationWizard(ApplicationWizardPermissionView, TemplateView): # by tyoing in review in the URL. The submit button still shows, hence # the if/else. if self.application.requested_domain: - modal_heading = 'You are about to submit a domain request for ' + str(self.application.requested_domain) + modal_heading = "You are about to submit a domain request for " + str(self.application.requested_domain) else: - modal_heading = 'You are about to submit an incomplete request' + modal_heading = "You are about to submit an incomplete request" return { "form_titles": self.TITLES, "steps": self.steps, From d5f55ba383735cc07bb7d531888fba9add081fae Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 15 Dec 2023 11:43:20 -0500 Subject: [PATCH 77/78] Clean up comments --- .../tests/test_copy_names_from_contacts_to_users.py | 2 +- src/registrar/views/application.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py index 988b57a4b..032203f4e 100644 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ b/src/registrar/tests/test_copy_names_from_contacts_to_users.py @@ -19,7 +19,7 @@ class TestDataUpdates(TestCase): self.user3 = User.objects.create(username="user3") self.user4 = User.objects.create(username="user4") # The last user created triggers the creation of a contact and attaches itself to it. See signals. - # This bs_user defuses that situation (unwanted user-contact pairing) so we can test the code. + # This bs_user defuses that situation. self.bs_user = User.objects.create() self.contact1 = Contact.objects.create( diff --git a/src/registrar/views/application.py b/src/registrar/views/application.py index ba716c117..0a6eb5b7b 100644 --- a/src/registrar/views/application.py +++ b/src/registrar/views/application.py @@ -321,15 +321,9 @@ class ApplicationWizard(ApplicationWizardPermissionView, TemplateView): def get_context_data(self): """Define context for access on all wizard pages.""" - # The on-page submit button is just a trigger for the modal; - # the submit button we're adding to context will get passed to - # the modal and is the button that triggers the actual domain - # application submission (via post -> goto_next_step -> done). + # Build the submit button that we'll pass to the modal. modal_button = '" - # We'll concatenate the modal header here for passing along to the - # modal include. NOTE: We are able to 'fast-forward' a domain application - # by tyoing in review in the URL. The submit button still shows, hence - # the if/else. + # Concatenate the modal header that we'll pass to the modal. if self.application.requested_domain: modal_heading = "You are about to submit a domain request for " + str(self.application.requested_domain) else: From 8479351e8aa2f5415e7e2ae560fd1349c79a7bb9 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Mon, 18 Dec 2023 12:56:15 -0600 Subject: [PATCH 78/78] Add missing usa-list class to a ul --- src/registrar/templates/domain_dns.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_dns.html b/src/registrar/templates/domain_dns.html index 0f625e0e3..291319a59 100644 --- a/src/registrar/templates/domain_dns.html +++ b/src/registrar/templates/domain_dns.html @@ -12,7 +12,7 @@

You can enter your name servers, as well as other DNS-related information, in the following sections:

{% url 'domain-dns-nameservers' pk=domain.id as url %} -
    +
    • Name servers
      • {% url 'domain-dns-dnssec' pk=domain.id as url %}
Your domain applicationsYour domain requests
Domain name