From d05c3c31a99673735f4412539de6b3f582f89d80 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:04:39 -0600 Subject: [PATCH 01/10] Update csv_export.py --- src/registrar/utility/csv_export.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 334742d17..dbd1bf5cb 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -18,6 +18,7 @@ from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region from registrar.utility.constants import BranchChoices +from registrar.utility.enums import DefaultEmail logger = logging.getLogger(__name__) @@ -371,6 +372,15 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + security_contact_email = model.get("security_contact_email") + invalid_emails = {DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT} + if ( + not security_contact_email + or not isinstance(security_contact_email, str) + or security_contact_email.lower() in invalid_emails + ): + security_contact_email = "(blank)" + # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { From 7b14b3d27b19eb718c4df4060422e5ceacf87e02 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:04:47 -0600 Subject: [PATCH 02/10] Revert "Update csv_export.py" This reverts commit d05c3c31a99673735f4412539de6b3f582f89d80. --- src/registrar/utility/csv_export.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index dbd1bf5cb..334742d17 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -18,7 +18,6 @@ from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region from registrar.utility.constants import BranchChoices -from registrar.utility.enums import DefaultEmail logger = logging.getLogger(__name__) @@ -372,15 +371,6 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - security_contact_email = model.get("security_contact_email") - invalid_emails = {DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT} - if ( - not security_contact_email - or not isinstance(security_contact_email, str) - or security_contact_email.lower() in invalid_emails - ): - security_contact_email = "(blank)" - # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { From deabe3514b109bc2e23eee46f443f8311f63d564 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:06:14 -0600 Subject: [PATCH 03/10] Update csv_export.py --- src/registrar/utility/csv_export.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 334742d17..dbd1bf5cb 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -18,6 +18,7 @@ from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region from registrar.utility.constants import BranchChoices +from registrar.utility.enums import DefaultEmail logger = logging.getLogger(__name__) @@ -371,6 +372,15 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + security_contact_email = model.get("security_contact_email") + invalid_emails = {DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT} + if ( + not security_contact_email + or not isinstance(security_contact_email, str) + or security_contact_email.lower() in invalid_emails + ): + security_contact_email = "(blank)" + # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { From 824be437fddedabd4bfb964d3549fbf29a6b6838 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:14:06 -0600 Subject: [PATCH 04/10] add strip --- src/registrar/utility/csv_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index dbd1bf5cb..f82fcbc99 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -377,7 +377,7 @@ class DomainExport(BaseExport): if ( not security_contact_email or not isinstance(security_contact_email, str) - or security_contact_email.lower() in invalid_emails + or security_contact_email.lower().strip() in invalid_emails ): security_contact_email = "(blank)" From 1433c2c31fc24fff4f1b92f41d320bdf604209a5 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:19:32 -0600 Subject: [PATCH 05/10] Update csv_export.py --- src/registrar/utility/csv_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index f82fcbc99..d53d3ec1d 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -373,7 +373,7 @@ class DomainExport(BaseExport): domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" security_contact_email = model.get("security_contact_email") - invalid_emails = {DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT} + invalid_emails = [DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT] if ( not security_contact_email or not isinstance(security_contact_email, str) From bdec481f8d84284c3fc404b3ff5588d2d4485f48 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:22:25 -0600 Subject: [PATCH 06/10] use .value --- src/registrar/utility/csv_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index d53d3ec1d..ad589f049 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -373,7 +373,7 @@ class DomainExport(BaseExport): domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" security_contact_email = model.get("security_contact_email") - invalid_emails = [DefaultEmail.LEGACY_DEFAULT, DefaultEmail.PUBLIC_CONTACT_DEFAULT] + invalid_emails = [DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value] if ( not security_contact_email or not isinstance(security_contact_email, str) From 0a592daaefd93e94cf7a31bcdb5711087a2e3ef5 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:26:38 -0600 Subject: [PATCH 07/10] Update csv_export.py --- src/registrar/utility/csv_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index ad589f049..6e378ed45 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -395,7 +395,7 @@ class DomainExport(BaseExport): "State": model.get("state_territory"), "SO": model.get("so_name"), "SO email": model.get("senior_official__email"), - "Security contact email": model.get("security_contact_email"), + "Security contact email": security_contact_email, "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), "Domain managers": model.get("managers"), From 7dd32baf5ed686f421d24e635a0111b6b232345d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:05:37 -0600 Subject: [PATCH 08/10] Update test_reports.py --- src/registrar/tests/test_reports.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index ded04e31b..2fd61eb53 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -47,10 +47,10 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -69,12 +69,12 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), - call("adomain2.gov,Interstate,,,,,\r\n"), - call("zdomain12.gov,Interstate,,,,,\r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), + call("adomain2.gov,Interstate,,,,,(blank)\r\n"), + call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. From db5333b37234ec885fae7cb4909741fc5650a215 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:09:56 -0600 Subject: [PATCH 09/10] fix tests --- src/registrar/tests/test_reports.py | 43 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 2fd61eb53..ce0929e77 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -233,29 +233,30 @@ class ExportDataTest(MockDb, MockEppLib): expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," "SO email,Security contact email,Domain managers,Invited domain managers\n" - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," "meoward@rocks.com,\n" "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," - ', ,,dotgov@cisa.dhs.gov,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + ',,,(blank),"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' "woofwardthethird@rocks.com\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,,," + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,, ,," + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," "security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,registrar@dotgov.gov," + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," "meoward@rocks.com,squeaker@rocks.com\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,,meoward@rocks.com,\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -285,17 +286,18 @@ class ExportDataTest(MockDb, MockEppLib): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" - "adomain2.gov,Interstate,,,,,registrar@dotgov.gov\n" - "zdomain12.gov,Interstate,,,,,\n" + "adomain2.gov,Interstate,,,,,(blank)\n" + "zdomain12.gov,Interstate,,,,,(blank)\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -325,15 +327,16 @@ class ExportDataTest(MockDb, MockEppLib): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator From f9bca3950655985042698847f72b5b2c88671b65 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:13:32 -0600 Subject: [PATCH 10/10] Update csv_export.py --- src/registrar/utility/csv_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 6e378ed45..56aa41569 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -373,7 +373,7 @@ class DomainExport(BaseExport): domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" security_contact_email = model.get("security_contact_email") - invalid_emails = [DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value] + invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} if ( not security_contact_email or not isinstance(security_contact_email, str)