diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 262cebd18..5bf41777b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -374,8 +374,8 @@ def analytics(request): avg_approval_time = last_30_days_approved_applications.annotate( approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) + # Format the timedelta to display only days + avg_approval_time = f"{avg_approval_time.days} days" start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -383,75 +383,69 @@ def analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) - # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - # Ready and Deleted domains filter_ready_domains_start_date = { "domain__state__in": [Domain.State.READY], "domain__first_ready__lte": start_date_formatted, } + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": start_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - - filter_ready_domains_end_date = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - } - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) - filter_deleted_domains_end_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": start_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - - filter_requests_end_date = { - "created_at__lte": end_date_formatted, - } - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) - filter_submitted_requests_end_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( @@ -459,6 +453,7 @@ def analytics(request): data=dict( user_count=models.User.objects.all().count(), domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(), last_30_days_applications=last_30_days_applications.count(), last_30_days_approved_applications=last_30_days_approved_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, @@ -1096,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1448,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] + readonly_fields = ["state", "expiration_date", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index dad88b6a4..29d0e3b2a 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -319,6 +319,9 @@ input.admin-confirm-button { .usa-icon { top: 2px; } + a.button:active, a.button:focus { + text-decoration: none; + } } .module--custom { diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..3b18ac8b6 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper): first_ready = DateField( null=True, - editable=False, + editable=True, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 29faffd3b..380922845 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -16,6 +16,7 @@
-
- -
+
+ +
diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index c96f29a31..4ee2befef 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -71,5 +71,5 @@

Analytics

- Dashboard + Dashboard
\ No newline at end of file diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py new file mode 100644 index 000000000..e6dccb14f --- /dev/null +++ b/src/registrar/tests/data/mocks.py @@ -0,0 +1,232 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation +from registrar.models.domain import Domain +from registrar.models.user_domain_role import UserDomainRole +from registrar.models.public_contact import PublicContact +from registrar.models.user import User +from datetime import date, datetime, timedelta +from django.utils import timezone +from registrar.tests.common import MockEppLib + +class MockDb(MockEppLib): + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + # Test for more than 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + # Test for just 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + # self.domain_request_1, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # requested_domain=self.domain_1.name, + # organization_type="federal", + # federal_agency="World War I Centennial Commission", + # federal_type="executive", + # is_election_board=True + # ) + # self.domain_request_2, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_2, + # organization_type="interstate", + # is_election_board=True + # ) + # self.domain_request_3, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_3, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_4, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_4, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_5, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_5, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_6, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_6, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_7, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_7, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_8, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_8, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_9, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_9, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_10, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_10, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + + def tearDown(self): + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index e55175db9..cc4b3f1c7 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -3,7 +3,7 @@ from django.urls import reverse from registrar.tests.common import create_superuser -class TestViews(TestCase): +class TestAdminViews(TestCase): def setUp(self): self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -26,7 +26,7 @@ class TestViews(TestCase): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index c00c2b221..43efb3128 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -8,9 +8,12 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.common import MockEppLib +from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - write_csv, + format_end_date, + format_start_date, + get_sliced_domains, + write_domains_csv, get_default_start_date, get_default_end_date, ) @@ -231,136 +234,11 @@ class CsvReportsTest(TestCase): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockEppLib): +class ExportDataTest(MockDb): def setUp(self): super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() super().tearDown() def test_export_domains_to_writer_security_emails(self): @@ -403,7 +281,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) @@ -427,7 +305,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_csv(self): + def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" @@ -462,7 +340,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -486,7 +364,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_additional(self): + def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" with less_console_noise(): # Create a CSV file in memory @@ -512,7 +390,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -535,7 +413,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_with_date_filter_pulls_domains_in_range(self): + def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): """Test that domains that are 1. READY and their first_ready dates are in range 2. DELETED and their deleted dates are in range @@ -546,7 +424,7 @@ class ExportDataTest(MockEppLib): and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. - TODO: Simplify is created_at is not needed for the report.""" + TODO: Simplify if created_at is not needed for the report.""" with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -591,7 +469,7 @@ class ExportDataTest(MockEppLib): } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -599,7 +477,7 @@ class ExportDataTest(MockEppLib): get_domain_managers=False, should_write_header=True, ) - write_csv( + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -664,7 +542,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True ) @@ -677,11 +555,11 @@ class ExportDataTest(MockEppLib): expected_content = ( "Domain name,Status,Expiration date,Domain type,Agency," "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager email 1,Domain manager email 2,\n" + "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,info@example.com\n" + ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" ) # Normalize line endings and remove commas, @@ -690,6 +568,210 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + def test_export_data_managed_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + # Call the export functions + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,1\n" + "\n" + "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" + "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_export_data_unmanaged_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # Call the export functions + writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=False, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): + """Test that requests that are + 1. SUBMITTED and their submission_date are in range + are pulled when the growth report conditions are applied to export_requests_to_writed. + Test that requests are sorted by requested domain name. + """ + + pass class HelperFunctions(TestCase): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -704,3 +786,11 @@ class HelperFunctions(TestCase): expected_date = timezone.now() actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) + + def get_sliced_domains(self): + """Should get fitered domains counts sliced by org type and election office.""" + pass + + def test_get_sliced_requests(self): + """Should get fitered requests counts sliced by org type and election office.""" + pass \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index bec5f3835..cbdbfddb3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -25,9 +25,10 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") + DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) + .distinct() ) # Do a mass concat of the first and last name fields for authorizing_official. @@ -44,7 +45,7 @@ def get_domain_infos(filter_condition, sort_fields): return domain_infos_cleaned -def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): +def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" # Domain should never be none when parsing this information @@ -136,7 +137,7 @@ def update_columns_with_domain_managers(columns, max_dm_count): columns.append(f"Domain manager email {i}") -def write_csv( +def write_domains_csv( writer, columns, sort_fields, @@ -145,8 +146,8 @@ def write_csv( should_write_header=True, ): """ - Receives params from the parent methods and outputs a CSV with fltered and sorted domains. - Works with write_header as longas the same writer object is passed. + Receives params from the parent methods and outputs a CSV with filtered and sorted domains. + Works with write_header as long as the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ @@ -172,7 +173,7 @@ def write_csv( rows = [] for domain_info in page.object_list: try: - row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers) + row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers) rows.append(row) except ValueError: # This should not happen. If it does, just skip this row. @@ -188,7 +189,6 @@ def write_csv( def get_requests(filter_condition, sort_fields): requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) - return requests @@ -235,7 +235,8 @@ def write_requests_csv( filter_condition, should_write_header=True, ): - """ """ + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" all_requetsts = get_requests(filter_condition, sort_fields) @@ -295,7 +296,7 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) def export_data_full_to_csv(csv_file): @@ -326,7 +327,7 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def export_data_federal_to_csv(csv_file): @@ -358,7 +359,7 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def get_default_start_date(): @@ -426,8 +427,8 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - write_csv( + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -440,19 +441,19 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): def get_sliced_domains(filter_condition): """Get fitered domains counts sliced by org type and election office.""" - domains = DomainInformation.objects.all().filter(**filter_condition) + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() domains_count = domains.count() - federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() state_or_territory = domains.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = domains.filter(is_election_board=True).count() + ).distinct().count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = domains.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -471,19 +472,19 @@ def get_sliced_domains(filter_condition): def get_sliced_requests(filter_condition): """Get fitered requests counts sliced by org type and election office.""" - requests = DomainApplication.objects.all().filter(**filter_condition) + requests = DomainApplication.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() - interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = requests.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = requests.filter(is_election_board=True).count() + ).distinct().count() + tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = requests.filter(is_election_board=True).distinct().count() return [ requests_count, @@ -500,7 +501,8 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get domains have domain managers for two different dates.""" + """Get counts for domains that have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -512,14 +514,13 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): sort_fields = [ "domain__name", ] - filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"]) + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( [ "Total", @@ -537,16 +538,6 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_managed_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, @@ -571,7 +562,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -582,7 +573,8 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get domains that do not have domain managers for two different dates.""" + """Get counts for domains that do not have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -619,16 +611,6 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, @@ -653,18 +635,23 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_unmanaged_domains_end_date, - get_domain_managers=True, + get_domain_managers=False, should_write_header=True, ) def export_data_requests_growth_to_csv(csv_file, start_date, end_date): - """ """ + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_requests_body SUBMITTED requests that are created between + the start and end dates. Specify sort params. + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -676,7 +663,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date): "Submission date", ] sort_fields = [ - # "domain__name", + "requested_domain__name", ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED,