diff --git a/src/registrar/management/commands/generate_current_federal_report.py b/src/registrar/management/commands/generate_current_federal_report.py index 6516bf99b..97d4fd7e4 100644 --- a/src/registrar/management/commands/generate_current_federal_report.py +++ b/src/registrar/management/commands/generate_current_federal_report.py @@ -50,7 +50,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_federal_to_csv(file) + csv_export.DomainDataFederal.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/management/commands/generate_current_full_report.py b/src/registrar/management/commands/generate_current_full_report.py index be810ee10..4bcb9f502 100644 --- a/src/registrar/management/commands/generate_current_full_report.py +++ b/src/registrar/management/commands/generate_current_full_report.py @@ -49,7 +49,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_full_to_csv(file) + csv_export.DomainDataFull.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 0028034fb..0015ae84f 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -1,21 +1,23 @@ -import csv import io from django.test import Client, RequestFactory from io import StringIO from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain -from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.utility.csv_export import ( - export_data_managed_domains_to_csv, - export_data_unmanaged_domains_to_csv, - get_sliced_domains, - get_sliced_requests, - write_csv_for_domains, + DomainDataFull, + DomainDataType, + DomainDataFederal, + DomainGrowth, + DomainManaged, + DomainUnmanaged, + DomainExport, + DomainRequestExport, + DomainRequestGrowth, + DomainRequestDataFull, get_default_start_date, get_default_end_date, - DomainRequestExport, ) - +from django.db.models import Case, When from django.core.management import call_command from unittest.mock import MagicMock, call, mock_open, patch from api.views import get_current_federal, get_current_full @@ -45,10 +47,10 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -67,11 +69,12 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("adomain2.gov,Interstate,,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("adomain2.gov,Interstate,,,,,\r\n"), + call("zdomain12.gov,Interstate,,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -202,494 +205,299 @@ class ExportDataTest(MockDb, MockEppLib): def tearDown(self): super().tearDown() - def test_export_domains_to_writer_security_emails_and_first_ready(self): - """Test that export_domains_to_writer returns the - expected security email and first_ready value""" + @less_console_noise_decorator + def test_domain_data_type(self): + """Shows security contacts, domain managers, ao""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataType.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO," + "AO email,Security contact email,Domain managers,Invited domain managers\n" + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," + "meoward@rocks.com,\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," + ', ,,dotgov@cisa.dhs.gov,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,,," + "squeaker@rocks.com\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,, ,," + "security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,registrar@dotgov.gov," + "meoward@rocks.com,squeaker@rocks.com\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): - # Add security email information - self.domain_1.name = "defaultsecurity.gov" - self.domain_1.save() - # Invoke setter - self.domain_1.security_contact - # Invoke setter - self.domain_2.security_contact - # Invoke setter - self.domain_3.security_contact + @less_console_noise_decorator + def test_domain_data_full(self): + """Shows security contacts, filtered by state""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "adomain2.gov,Interstate,,,,,registrar@dotgov.gov\n" + "zdomain12.gov,Interstate,,,,,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Add a first ready date on the first domain. Leaving the others blank. - self.domain_1.first_ready = get_default_start_date() - self.domain_1.save() + @less_console_noise_decorator + def test_domain_data_federal(self): + """Shows security contacts, filtered by state and org type""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFederal.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - "Status", - "Expiration date", - "First ready on", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, + @less_console_noise_decorator + def test_domain_growth(self): + """Shows ready and deleted domains within a date range, sorted""" + # Remove "Created at" and "First ready" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + # "Created at", + # "First ready", + "Deleted", + ] + sort = { + "custom_sort": Case( + When(domain__state=Domain.State.READY, then="domain__created_at"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), ) + } + with patch("registrar.utility.csv_export.DomainGrowth.get_columns", return_value=columns): + with patch("registrar.utility.csv_export.DomainGrowth.get_annotations_for_sort", return_value=sort): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name + # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City," + "State,Status,Expiration date, Deleted\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" + "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" + "zdomain12.govInterstateReady(blank)\n" + "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = ( + csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + ) + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Security contact email,Status,Expiration date, First ready on\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready,(blank),2024-04-03\n" - "adomain2.gov,Interstate,(blank),Dns needed,(blank),(blank)\n" - "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank),2024-04-02\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,security@mail.gov,On hold,2023-11-15,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission," - "(blank),Ready,(blank),2023-11-01\n" - "zdomain12.govInterstateReady,(blank),2024-04-02\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_csv_for_domains(self): - """Test that write_body returns the - existing domain, test that sort by domain name works, - test that filter works""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Submitter", - "Submitter title", - "Submitter email", - "Submitter phone", - "Security contact email", - "Status", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Submitter,Submitter title,Submitter email,Submitter phone," - "Security contact email,Status\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" - "adomain2.gov,Interstate,Dns needed\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n" - "zdomain12.govInterstateReady\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_additional(self): - """An additional test for filters and multi-column sort""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - sort_fields = ["domain__name", "federal_agency", "generic_org_type"] - filter_condition = { - "generic_org_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # federal only - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Security contact email\n" - "adomain10.gov,Federal,Armed Forces Retirement Home\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommission\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): - """Test that domains that are - 1. READY and their first_ready dates are in range - 2. DELETED and their deleted dates are in range - are pulled when the growth report conditions are applied to export_domains_to_writed. - Test that ready domains are sorted by first_ready/deleted dates first, names second. - - We considered testing export_data_domain_growth_to_csv which calls write_body - and would have been easy to set up, but expected_content would contain created_at dates - which are hard to mock. - - TODO: Simplify if created_at is not needed for the report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - ] - sort_fields = [ - "created_at", - "domain__name", - ] - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - ], - "domain__first_ready__lte": self.end_date, - "domain__first_ready__gte": self.start_date, - } - filter_conditions_for_deleted_domains = { - "domain__state__in": [ - Domain.State.DELETED, - ], - "domain__deleted__lte": self.end_date, - "domain__deleted__gte": self.start_date, - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - write_csv_for_domains( - writer, - columns, - sort_fields_for_deleted_domains, - filter_conditions_for_deleted_domains, - should_get_domain_managers=False, - should_write_header=False, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - - # Read the content into a variable - csv_content = csv_file.read() - - # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name - # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Status,Expiration date\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" - "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank)\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank)\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank)\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_domains_to_writer_domain_managers(self): - """Test that export_domains_to_writer returns the - expected domain managers. + @less_console_noise_decorator + def test_domain_managed(self): + """Shows ready and deleted domains by an end date, sorted An invited user, woofwardthethird, should also be pulled into this report. squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). - She should show twice in this report but not in test_export_data_managed_domains_to_csv.""" + She should show twice in this report but not in test_DomainManaged.""" + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainManaged.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" + "3,2,1,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type,Domain managers,Invited domain managers\n" + "cdomain11.gov,Federal - Executive,meoward@rocks.com,\n" + 'cdomain1.gov,Federal - Executive,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "zdomain12.gov,Interstate,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): + @less_console_noise_decorator + def test_domain_unmanaged(self): + """Shows unmanaged domains by an end date, sorted""" + # Create a CSV file in memory + csv_file = StringIO() + DomainUnmanaged.export_data_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + @less_console_noise_decorator + def test_domain_request_growth(self): + """Shows submitted requests within a date range, sorted""" + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + "Domain type", + "Federal type", + # "Submitted at", + ] + with patch("registrar.utility.csv_export.DomainRequestGrowth.get_columns", return_value=columns): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Status", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=True, - should_write_header=True, + DomainRequestGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), ) - # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Status,Expiration date,Domain type,Agency," - "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "adomain10.gov,Ready,(blank),Federal,Armed Forces Retirement Home,,,, , ,squeaker@rocks.com, I\n" - "adomain2.gov,Dns needed,(blank),Interstate,,,,, , , ,meoward@rocks.com, R,squeaker@rocks.com, I\n" - "cdomain11.govReady,(blank),Federal-ExecutiveWorldWarICentennialCommissionmeoward@rocks.comR\n" - "cdomain1.gov,Ready,(blank),Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,R,info@example.com,R,big_lebowski@dude.co,R," - "woofwardthethird@rocks.com,I\n" - "ddomain3.gov,On hold,(blank),Federal,Armed Forces Retirement Home,,,, , , ,,\n" - "zdomain12.gov,Ready,(blank),Interstate,meoward@rocks.com,R\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_export_data_managed_domains_to_csv(self): - """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date. - - An invited user, woofwardthethird, should also be pulled into this report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_managed_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City," - "Special district,School district,Election office\n" - "3,2,1,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "cdomain11.govFederal-Executivemeoward@rocks.com, R\n" - "cdomain1.gov,Federal - Executive,meoward@rocks.com,R,info@example.com,R," - "big_lebowski@dude.co,R,woofwardthethird@rocks.com,I\n" - "zdomain12.govInterstatemeoward@rocks.com,R\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_data_unmanaged_domains_to_csv(self): - """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_unmanaged_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "UNMANAGED DOMAINS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "UNMANAGED DOMAINS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "1,1,0,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type\n" - "adomain10.gov,Federal\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): - """Test that requests that are - 1. SUBMITTED and their submission_date are in range - are pulled when the growth report conditions are applied to export_requests_to_writed. - Test that requests are sorted by requested domain name. - """ - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - # We'll skip submission date because it's dynamic and therefore - # impossible to set in expected_content - columns = ["Domain request", "Domain type", "Federal type"] - sort_fields = [ - "requested_domain__name", - ] - filter_condition = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": self.end_date, - "submission_date__gte": self.start_date, - } - - additional_values = ["requested_domain__name"] - all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(all_requests, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, columns, requests_dict) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name - # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( "Domain request,Domain type,Federal type\n" "city3.gov,Federal,Executive\n" @@ -705,70 +513,82 @@ class ExportDataTest(MockDb, MockEppLib): self.assertEqual(csv_content, expected_content) @less_console_noise_decorator - def test_full_domain_request_report(self): + def test_domain_request_data_full(self): """Tests the full domain request report.""" - - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Call the report. Get existing fields from the report itself. - annotations = DomainRequestExport._full_domain_request_annotations() - additional_values = [ - "requested_domain__name", - "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", - "creator__first_name", - "creator__last_name", - "creator__email", - "investigator__email", + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + # "Submitted at", + "Status", + "Domain type", + "Federal type", + "Federal agency", + "Organization name", + "Election office", + "City", + "State/territory", + "Region", + "Creator first name", + "Creator last name", + "Creator email", + "Creator approved domains count", + "Creator active requests count", + "Alternative domains", + "AO first name", + "AO last name", + "AO email", + "AO title/role", + "Request purpose", + "Request additional details", + "Other contacts", + "CISA regional representative", + "Current websites", + "Investigator", ] - requests = DomainRequest.objects.exclude(status=DomainRequest.DomainRequestStatus.STARTED) - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, DomainRequestExport.all_columns, requests_dict) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - print(csv_content) - self.maxDiff = None - expected_content = ( - # Header - "Domain request,Submitted at,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," - "AO title/role,Request purpose,Request additional details,Other contacts," - "CISA regional representative,Current websites,Investigator\n" - # Content - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1," - "cheeseville.gov | city1.gov | igorville.gov,Testy,Tester,testy@town.com,Chief Tester," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Meow Tester24 te2@town.com | " - "Testy1232 Tester24 te2@town.com | Testy Tester testy2@town.com,test@igorville.com," - "city.com | https://www.example2.com | https://www.example.com,\n" - "city4.gov,2024-04-02,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,\n" - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city6.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com," - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) + with patch("registrar.utility.csv_export.DomainRequestDataFull.get_columns", return_value=columns): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainRequestDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + print(csv_content) + expected_content = ( + # Header + "Domain request,Status,Domain type,Federal type," + "Federal agency,Organization name,Election office,City,State/territory," + "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," + "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," + "AO title/role,Request purpose,Request additional details,Other contacts," + "CISA regional representative,Current websites,Investigator\n" + # Content + "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," + "testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' + 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' + "CISA-last-name " + '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' + 'testy2@town.com"' + ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com" + ",cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -794,12 +614,12 @@ class HelperFunctions(MockDb): "domain__first_ready__lte": self.end_date, } # Test with distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) # Test without distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) @@ -811,6 +631,6 @@ class HelperFunctions(MockDb): "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } - submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 6875f0e3d..f2786c2cd 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -15,12 +15,10 @@ from django.db.models import QuerySet, Value, CharField, Count, Q, F from django.db.models import Case, When, DateField from django.db.models import ManyToManyField from django.utils import timezone -from django.core.paginator import Paginator from django.db.models.functions import Concat, Coalesce from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region -from registrar.utility.enums import DefaultEmail from registrar.utility.constants import BranchChoices @@ -34,14 +32,17 @@ def write_header(writer, columns): """ writer.writerow(columns) + def get_default_start_date(): - # Default to a date that's prior to our first deployment + """Default to a date that's prior to our first deployment""" return timezone.make_aware(datetime(2023, 11, 1)) + def get_default_end_date(): - # Default to now() + """Default to now()""" return timezone.now() + def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() @@ -49,9 +50,11 @@ def format_start_date(start_date): def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + class BaseExport(ABC): """ A generic class for exporting data which returns a csv file for the given model. + Base class in an inheritance tree of 3. """ @classmethod @@ -69,14 +72,14 @@ class BaseExport(ABC): Returns the columns for CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields for the CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_additional_args(cls): """ @@ -84,63 +87,63 @@ class BaseExport(ABC): Override in subclasses to provide specific arguments. """ return {} - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ return [] - + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ return [] - + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ return Q() - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ Get a Q object of filter conditions to filter when building queryset. """ return Q() - + @classmethod def get_computed_fields(cls): """ Get a dict of computed fields. """ return {} - + @classmethod def get_annotations_for_sort(cls): """ Get a dict of annotations to make available for order_by clause. """ return {} - + @classmethod def get_related_table_fields(cls): """ Get a list of fields from related tables. """ return [] - + @classmethod def update_queryset(cls, queryset, **kwargs): """ Returns an updated queryset. Override in subclass to update queryset. """ return queryset - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -187,7 +190,7 @@ class BaseExport(ABC): queryset = initial_queryset.annotate(**computed_fields).values(*model_fields, *related_table_fields) return cls.update_queryset(queryset, **kwargs) - + @classmethod def export_data_to_csv(cls, csv_file, start_date=None, end_date=None): """ @@ -207,7 +210,8 @@ class BaseExport(ABC): related_table_fields = cls.get_related_table_fields() model_queryset = ( - cls.model().objects.select_related(*select_related) + cls.model() + .objects.select_related(*select_related) .prefetch_related(*prefetch_related) .filter(filter_conditions) .exclude(exclusions) @@ -217,7 +221,9 @@ class BaseExport(ABC): ) # Convert the queryset to a dictionary (including annotated fields) - annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) + annotated_queryset = cls.annotate_and_retrieve_fields( + model_queryset, computed_fields, related_table_fields, **kwargs + ) models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) # Write to csv file before the write_csv @@ -259,10 +265,11 @@ class BaseExport(ABC): """ pass - + class DomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. + Second class in an inheritance tree of 3. """ @classmethod @@ -279,9 +286,9 @@ class DomainExport(BaseExport): based on public_contacts, domain_invitations and user_domain_roles passed through kwargs. """ - public_contacts = kwargs.get('public_contacts', {}) - domain_invitations = kwargs.get('domain_invitations', {}) - user_domain_roles = kwargs.get('user_domain_roles', {}) + public_contacts = kwargs.get("public_contacts", {}) + domain_invitations = kwargs.get("domain_invitations", {}) + user_domain_roles = kwargs.get("user_domain_roles", {}) annotated_domain_infos = [] @@ -296,16 +303,18 @@ class DomainExport(BaseExport): # Annotate with security_contact from public_contacts for domain_info in queryset: - domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) - domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) - domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) + domain_info["security_contact_email"] = public_contacts.get( + domain_info.get("domain__security_contact_registry_id") + ) + domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) + domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) annotated_domain_infos.append(domain_info) if annotated_domain_infos: return annotated_domain_infos - + return queryset - + # ============================================================= # # Helper functions for django ORM queries. # # We are using these rather than pure python for speed reasons. # @@ -316,15 +325,15 @@ class DomainExport(BaseExport): """ Fetch all PublicContact entries and return a mapping of registry_id to email. """ - public_contacts = PublicContact.objects.values_list('registry_id', 'email') + public_contacts = PublicContact.objects.values_list("registry_id", "email") return {registry_id: email for registry_id, email in public_contacts} - + @classmethod def get_all_domain_invitations(cls): """ Fetch all DomainInvitation entries and return a mapping of domain to email. """ - domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") return list(domain_invitations) @classmethod @@ -332,7 +341,7 @@ class DomainExport(BaseExport): """ Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ - user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") return list(user_domain_roles) @classmethod @@ -360,19 +369,9 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - if model.get("domain__name") == "18f.gov": - print(f'domain_type {domain_type}') - print(f'federal_agency {model.get("federal_agency")}') - print(f'city {model.get("city")}') - - print(f'agency {model.get("agency")}') - - print(f'federal_agency__agency {model.get("federal_agency__agency")}') - # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { - "Domain name": model.get("domain__name"), "Status": human_readable_status, "First ready on": first_ready_on, @@ -434,6 +433,10 @@ class DomainExport(BaseExport): class DomainDataType(DomainExport): + """ + Shows security contacts, domain managers, ao + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -456,7 +459,7 @@ class DomainDataType(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ @@ -488,29 +491,24 @@ class DomainDataType(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'public_contacts': public_contacts, - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "public_contacts": public_contacts, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain", - "authorizing_official" - ] - + return ["domain", "authorizing_official"] + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] + return ["permissions"] @classmethod def get_computed_fields(cls, delimiter=", "): @@ -525,7 +523,7 @@ class DomainDataType(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -542,9 +540,13 @@ class DomainDataType(DomainExport): "authorizing_official__email", "federal_agency__agency", ] - + class DomainDataFull(DomainExport): + """ + Shows security contacts, filtered by state + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -560,7 +562,7 @@ class DomainDataFull(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -586,17 +588,15 @@ class DomainDataFull(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -604,13 +604,13 @@ class DomainDataFull(DomainExport): Get a Q object of filter conditions to filter when building queryset. """ return Q( - domain__state__in = [ + domain__state__in=[ Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -624,7 +624,7 @@ class DomainDataFull(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -638,6 +638,10 @@ class DomainDataFull(DomainExport): class DomainDataFederal(DomainExport): + """ + Shows security contacts, filtered by state and org type + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -653,7 +657,7 @@ class DomainDataFederal(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -679,17 +683,15 @@ class DomainDataFederal(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -702,9 +704,9 @@ class DomainDataFederal(DomainExport): Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, - ] + ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -718,7 +720,7 @@ class DomainDataFederal(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -732,6 +734,10 @@ class DomainDataFederal(DomainExport): class DomainGrowth(DomainExport): + """ + Shows ready and deleted domains within a date range, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -751,7 +757,7 @@ class DomainGrowth(DomainExport): "First ready", "Deleted", ] - + @classmethod def get_annotations_for_sort(cls, delimiter=", "): """ @@ -760,10 +766,10 @@ class DomainGrowth(DomainExport): today = timezone.now().date() return { "custom_sort": Case( - When(domain__state=Domain.State.READY, then='domain__first_ready'), - When(domain__state=Domain.State.DELETED, then='domain__deleted'), + When(domain__state=Domain.State.READY, then="domain__first_ready"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), default=Value(today), # Default value if no conditions match - output_field=DateField() + output_field=DateField(), ) } @@ -773,19 +779,17 @@ class DomainGrowth(DomainExport): Returns the sort fields. """ return [ - '-domain__state', - 'custom_sort', - 'domain__name', + "-domain__state", + "custom_sort", + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -795,15 +799,13 @@ class DomainGrowth(DomainExport): filter_ready = Q( domain__state__in=[Domain.State.READY], domain__first_ready__gte=start_date, - domain__first_ready__lte=end_date + domain__first_ready__lte=end_date, ) filter_deleted = Q( - domain__state__in=[Domain.State.DELETED], - domain__deleted__gte=start_date, - domain__deleted__lte=end_date + domain__state__in=[Domain.State.DELETED], domain__deleted__gte=start_date, domain__deleted__lte=end_date ) return filter_ready | filter_deleted - + @classmethod def get_related_table_fields(cls): """ @@ -821,6 +823,10 @@ class DomainGrowth(DomainExport): class DomainManaged(DomainExport): + """ + Shows managed domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -833,34 +839,30 @@ class DomainManaged(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -871,7 +873,6 @@ class DomainManaged(DomainExport): domain__permissions__isnull=False, domain__first_ready__lte=end_date_formatted, ) - @classmethod def get_additional_args(cls): @@ -889,10 +890,10 @@ class DomainManaged(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_related_table_fields(cls): """ @@ -901,7 +902,7 @@ class DomainManaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -959,6 +960,10 @@ class DomainManaged(DomainExport): class DomainUnmanaged(DomainExport): + """ + Shows unmanaged domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -969,34 +974,30 @@ class DomainUnmanaged(DomainExport): "Domain name", "Domain type", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1007,7 +1008,7 @@ class DomainUnmanaged(DomainExport): domain__permissions__isnull=True, domain__first_ready__lte=end_date_formatted, ) - + @classmethod def get_related_table_fields(cls): """ @@ -1016,12 +1017,12 @@ class DomainUnmanaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ Write to csv file before the write_csv method. - + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -1075,6 +1076,10 @@ class DomainUnmanaged(DomainExport): class DomainRequestExport(BaseExport): + """ + A collection of functions which return csv files regarding the DomainRequest model. + Second class in an inheritance tree of 3. + """ @classmethod def model(cls): @@ -1197,6 +1202,10 @@ class DomainRequestExport(BaseExport): class DomainRequestGrowth(DomainRequestExport): + """ + Shows submitted requests within a date range, sorted + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1218,7 +1227,7 @@ class DomainRequestGrowth(DomainRequestExport): return [ "requested_domain__name", ] - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1238,12 +1247,14 @@ class DomainRequestGrowth(DomainRequestExport): """ Get a list of fields from related tables. """ - return [ - "requested_domain__name" - ] - + return ["requested_domain__name"] + class DomainRequestDataFull(DomainRequestExport): + """ + Shows all but STARTED requests + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1285,34 +1296,22 @@ class DomainRequestDataFull(DomainRequestExport): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "creator", - "authorizing_official", - "federal_agency", - "investigator", - "requested_domain" - ] + return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "current_websites", - "other_contacts", - "alternative_domains" - ] - + return ["current_websites", "other_contacts", "alternative_domains"] + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ - return Q( - status__in=[DomainRequest.DomainRequestStatus.STARTED] - ) - + return Q(status__in=[DomainRequest.DomainRequestStatus.STARTED]) + @classmethod def get_sort_fields(cls): """ @@ -1322,7 +1321,7 @@ class DomainRequestDataFull(DomainRequestExport): "status", "requested_domain__name", ] - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -1346,7 +1345,7 @@ class DomainRequestDataFull(DomainRequestExport): distinct=True, ), } - + @classmethod def get_related_table_fields(cls): """ @@ -1364,7 +1363,6 @@ class DomainRequestDataFull(DomainRequestExport): "creator__email", "investigator__email", ] - # ============================================================= # # Helper functions for django ORM queries. # diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 63e8492b5..4d015ab37 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,7 +49,9 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_managed_domains_start_date + ) managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { @@ -60,8 +62,12 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_start_date + ) + unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_end_date + ) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -82,7 +88,9 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_deleted_domains_start_date + ) deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { @@ -102,8 +110,12 @@ class AnalyticsView(View): "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_start_date + ) + submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_end_date + ) context = dict( # Generate a dictionary of context variables that are common across all admin templates