From d1b24ab6a294fc441c3116fa8b605eab25aa9e37 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:48:11 -0700 Subject: [PATCH 1/3] Add PR changes --- src/registrar/admin.py | 1 + .../commands/load_transition_domain.py | 28 ++-- .../transfer_transition_domains_to_domains.py | 16 +- .../utility/extra_transition_domain_helper.py | 12 +- src/registrar/models/transition_domain.py | 6 + .../test_transition_domain_migrations.py | 149 ++++++++++++++++++ 6 files changed, 195 insertions(+), 17 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 429bd762f..3b8f7c962 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -752,6 +752,7 @@ class TransitionDomainAdmin(ListHeaderAdmin): "domain_name", "status", "email_sent", + "processed", ] search_fields = ["username", "domain_name"] diff --git a/src/registrar/management/commands/load_transition_domain.py b/src/registrar/management/commands/load_transition_domain.py index e1165bf9f..4132096c8 100644 --- a/src/registrar/management/commands/load_transition_domain.py +++ b/src/registrar/management/commands/load_transition_domain.py @@ -536,19 +536,27 @@ class Command(BaseCommand): domain_name=new_entry_domain_name, ) - if existing_entry.status != new_entry_status: - # DEBUG: + if not existing_entry.processed: + if existing_entry.status != new_entry_status: + TerminalHelper.print_conditional( + debug_on, + f"{TerminalColors.OKCYAN}" + f"Updating entry: {existing_entry}" + f"Status: {existing_entry.status} > {new_entry_status}" # noqa + f"Email Sent: {existing_entry.email_sent} > {new_entry_emailSent}" # noqa + f"{TerminalColors.ENDC}", + ) + existing_entry.status = new_entry_status + existing_entry.email_sent = new_entry_emailSent + existing_entry.save() + else: TerminalHelper.print_conditional( debug_on, - f"{TerminalColors.OKCYAN}" - f"Updating entry: {existing_entry}" - f"Status: {existing_entry.status} > {new_entry_status}" # noqa - f"Email Sent: {existing_entry.email_sent} > {new_entry_emailSent}" # noqa + f"{TerminalColors.YELLOW}" + f"Skipping update on processed domain: {existing_entry}" f"{TerminalColors.ENDC}", ) - existing_entry.status = new_entry_status - existing_entry.email_sent = new_entry_emailSent - existing_entry.save() + except TransitionDomain.MultipleObjectsReturned: logger.info( f"{TerminalColors.FAIL}" @@ -558,6 +566,7 @@ class Command(BaseCommand): f"----------TERMINATING----------" ) sys.exit() + else: # no matching entry, make one new_entry = TransitionDomain( @@ -565,6 +574,7 @@ class Command(BaseCommand): domain_name=new_entry_domain_name, status=new_entry_status, email_sent=new_entry_emailSent, + processed=False, ) to_create.append(new_entry) total_new_entries += 1 diff --git a/src/registrar/management/commands/transfer_transition_domains_to_domains.py b/src/registrar/management/commands/transfer_transition_domains_to_domains.py index d0d6ff363..15cd7376d 100644 --- a/src/registrar/management/commands/transfer_transition_domains_to_domains.py +++ b/src/registrar/management/commands/transfer_transition_domains_to_domains.py @@ -559,7 +559,8 @@ class Command(BaseCommand): debug_max_entries_to_parse, total_rows_parsed, ): - for transition_domain in TransitionDomain.objects.all(): + changed_transition_domains = TransitionDomain.objects.filter(processed=False) + for transition_domain in changed_transition_domains: ( target_domain_information, associated_domain, @@ -644,7 +645,8 @@ class Command(BaseCommand): debug_max_entries_to_parse, total_rows_parsed, ): - for transition_domain in TransitionDomain.objects.all(): + changed_transition_domains = TransitionDomain.objects.filter(processed=False) + for transition_domain in changed_transition_domains: # Create some local variables to make data tracing easier transition_domain_name = transition_domain.domain_name transition_domain_status = transition_domain.status @@ -796,6 +798,7 @@ class Command(BaseCommand): # First, save all Domain objects to the database Domain.objects.bulk_create(domains_to_create) + # DomainInvitation.objects.bulk_create(domain_invitations_to_create) # TODO: this is to resolve an error where bulk_create @@ -847,6 +850,15 @@ class Command(BaseCommand): ) DomainInformation.objects.bulk_create(domain_information_to_create) + # Loop through the list of everything created, and mark it as processed + for domain in domains_to_create: + name = domain.name + TransitionDomain.objects.filter(domain_name=name).update(processed=True) + + # Loop through the list of everything updated, and mark it as processed + for name in updated_domain_entries: + TransitionDomain.objects.filter(domain_name=name).update(processed=True) + self.print_summary_of_findings( domains_to_create, updated_domain_entries, diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 04170811f..54f68d5c8 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -155,13 +155,13 @@ class LoadExtraTransitionDomain: def update_transition_domain_models(self): """Updates TransitionDomain objects based off the file content given in self.parsed_data_container""" - all_transition_domains = TransitionDomain.objects.all() - if not all_transition_domains.exists(): - raise ValueError("No TransitionDomain objects exist.") + valid_transition_domains = TransitionDomain.objects.filter(processed=False) + if not valid_transition_domains.exists(): + raise ValueError("No updatable TransitionDomain objects exist.") updated_transition_domains = [] failed_transition_domains = [] - for transition_domain in all_transition_domains: + for transition_domain in valid_transition_domains: domain_name = transition_domain.domain_name updated_transition_domain = transition_domain try: @@ -228,7 +228,7 @@ class LoadExtraTransitionDomain: # DATA INTEGRITY CHECK # Make sure every Transition Domain got updated total_transition_domains = len(updated_transition_domains) - total_updates_made = TransitionDomain.objects.all().count() + total_updates_made = TransitionDomain.objects.filter(processed=False).count() if total_transition_domains != total_updates_made: # noqa here for line length logger.error( @@ -787,7 +787,7 @@ class OrganizationDataLoader: self.tds_to_update: List[TransitionDomain] = [] def update_organization_data_for_all(self): - """Updates org address data for all TransitionDomains""" + """Updates org address data for valid TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) == 0: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE) diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 28bdc4fc7..6fe230951 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -43,6 +43,12 @@ class TransitionDomain(TimeStampedModel): verbose_name="email sent", help_text="indicates whether email was sent", ) + processed = models.BooleanField( + null=False, + default=True, + verbose_name="Processed", + help_text="Indicates whether this TransitionDomain was already processed", + ) organization_type = models.TextField( max_length=255, null=True, diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 4e549bdd6..cfee68fea 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -21,6 +21,155 @@ from registrar.models.contact import Contact from .common import less_console_noise +class TestProcessedMigrations(TestCase): + """This test case class is designed to verify the idempotency of migrations + related to domain transitions in the application.""" + + def setUp(self): + """Defines the file name of migration_json and the folder its contained in""" + self.test_data_file_location = "registrar/tests/data" + self.migration_json_filename = "test_migrationFilepaths.json" + self.user, _ = User.objects.get_or_create(username="igorvillian") + + def tearDown(self): + """Deletes all DB objects related to migrations""" + # Delete domain information + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainInvitation.objects.all().delete() + TransitionDomain.objects.all().delete() + + # Delete users + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def run_load_domains(self): + """ + This method executes the load_transition_domain command. + + It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, + which is a user prompt in the terminal. The mock function always returns True, + allowing the test to proceed without manual user input. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + # noqa here because splitting this up makes it confusing. + # ES501 + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_transition_domain", + self.migration_json_filename, + directory=self.test_data_file_location, + ) + + def run_transfer_domains(self): + """ + This method executes the transfer_transition_domains_to_domains command. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ + call_command("transfer_transition_domains_to_domains") + + def test_domain_idempotent(self): + """ + This test ensures that the domain transfer process + is idempotent on Domain and DomainInformation. + """ + unchanged_domain, _ = Domain.objects.get_or_create( + name="testdomain.gov", + state=Domain.State.READY, + expiration_date=datetime.date(2000, 1, 1), + ) + unchanged_domain_information, _ = DomainInformation.objects.get_or_create( + domain=unchanged_domain, organization_name="test org name", creator=self.user + ) + self.run_load_domains() + + # Test that a given TransitionDomain isn't set to "processed" + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertFalse(transition_domain_object.processed) + + self.run_transfer_domains() + + # Test that old data isn't corrupted + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Test that a given TransitionDomain is set to "processed" after we transfer domains + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertTrue(transition_domain_object.processed) + + # Manually change Domain/DomainInformation objects + changed_domain = Domain.objects.filter(name="fakewebsite3.gov").get() + changed_domain.expiration_date = datetime.date(1999, 1, 1) + + changed_domain.save() + + changed_domain_information = DomainInformation.objects.filter(domain=changed_domain).get() + changed_domain_information.organization_name = "changed" + + changed_domain_information.save() + + # Rerun transfer domains + self.run_transfer_domains() + + # Test that old data isn't corrupted after running this twice + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Ensure that domain hasn't changed + actual_domain = Domain.objects.filter(name="fakewebsite3.gov").get() + self.assertEqual(changed_domain, actual_domain) + + # Ensure that DomainInformation hasn't changed + actual_domain_information = DomainInformation.objects.filter(domain=changed_domain).get() + self.assertEqual(changed_domain_information, actual_domain_information) + + def test_transition_domain_is_processed(self): + """ + This test checks if a domain is correctly marked as processed in the transition. + """ + old_transition_domain, _ = TransitionDomain.objects.get_or_create(domain_name="testdomain.gov") + # Asser that old records default to 'True' + self.assertTrue(old_transition_domain.processed) + + unchanged_domain, _ = Domain.objects.get_or_create( + name="testdomain.gov", + state=Domain.State.READY, + expiration_date=datetime.date(2000, 1, 1), + ) + unchanged_domain_information, _ = DomainInformation.objects.get_or_create( + domain=unchanged_domain, organization_name="test org name", creator=self.user + ) + self.run_load_domains() + + # Test that a given TransitionDomain isn't set to "processed" + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertFalse(transition_domain_object.processed) + + self.run_transfer_domains() + + # Test that old data isn't corrupted + actual_unchanged = Domain.objects.filter(name="testdomain.gov").get() + actual_unchanged_information = DomainInformation.objects.filter(domain=actual_unchanged).get() + self.assertEqual(unchanged_domain, actual_unchanged) + self.assertTrue(old_transition_domain.processed) + self.assertEqual(unchanged_domain_information, actual_unchanged_information) + + # Test that a given TransitionDomain is set to "processed" after we transfer domains + transition_domain_object = TransitionDomain.objects.get(domain_name="fakewebsite3.gov") + self.assertTrue(transition_domain_object.processed) + + class TestOrganizationMigration(TestCase): def setUp(self): """Defines the file name of migration_json and the folder its contained in""" From 80e1ad85e853906c9f23250cace5c418684b9732 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:46:56 -0700 Subject: [PATCH 2/3] Add migration --- .../0055_transitiondomain_processed.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/registrar/migrations/0055_transitiondomain_processed.py diff --git a/src/registrar/migrations/0055_transitiondomain_processed.py b/src/registrar/migrations/0055_transitiondomain_processed.py new file mode 100644 index 000000000..a2fb78edd --- /dev/null +++ b/src/registrar/migrations/0055_transitiondomain_processed.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.7 on 2023-12-12 21:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0054_alter_domainapplication_federal_agency_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="transitiondomain", + name="processed", + field=models.BooleanField( + default=True, + help_text="Indicates whether this TransitionDomain was already processed", + verbose_name="Processed", + ), + ), + ] From d89ace47ba2f7c35c371502620f4cdd8403d0a38 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:53:12 -0700 Subject: [PATCH 3/3] Fix migration weirdness --- ...ter_domain_state_alter_domainapplication_status_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/registrar/migrations/{0055_alter_domain_state_alter_domainapplication_status_and_more.py => 0056_alter_domain_state_alter_domainapplication_status_and_more.py} (96%) diff --git a/src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py b/src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py similarity index 96% rename from src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py rename to src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py index 9b6bac48c..097cddf8a 100644 --- a/src/registrar/migrations/0055_alter_domain_state_alter_domainapplication_status_and_more.py +++ b/src/registrar/migrations/0056_alter_domain_state_alter_domainapplication_status_and_more.py @@ -6,7 +6,7 @@ import django_fsm class Migration(migrations.Migration): dependencies = [ - ("registrar", "0054_alter_domainapplication_federal_agency_and_more"), + ("registrar", "0055_transitiondomain_processed"), ] operations = [