Refactor get_sliced_domains to get accurate numbers

This commit is contained in:
Rachid Mrad 2024-03-25 12:14:10 -04:00
parent 7bb3439c3e
commit cedd54c673
No known key found for this signature in database
3 changed files with 60 additions and 15 deletions

View file

@ -548,6 +548,9 @@ class MockDb(TestCase):
state=Domain.State.READY, state=Domain.State.READY,
first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())),
) )
self.domain_11, _ = Domain.objects.get_or_create(
name="cdomain11.gov", state=Domain.State.READY, first_ready=timezone.now()
)
self.domain_information_1, _ = DomainInformation.objects.get_or_create( self.domain_information_1, _ = DomainInformation.objects.get_or_create(
creator=self.user, creator=self.user,
@ -616,6 +619,14 @@ class MockDb(TestCase):
federal_agency="Armed Forces Retirement Home", federal_agency="Armed Forces Retirement Home",
is_election_board=False, is_election_board=False,
) )
self.domain_information_11, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_11,
generic_org_type="federal",
federal_agency="World War I Centennial Commission",
federal_type="executive",
is_election_board=True,
)
meoward_user = get_user_model().objects.create( meoward_user = get_user_model().objects.create(
username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com"
@ -641,6 +652,10 @@ class MockDb(TestCase):
user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER
) )
_, created = UserDomainRole.objects.get_or_create(
user=meoward_user, domain=self.domain_11, role=UserDomainRole.Roles.MANAGER
)
with less_console_noise(): with less_console_noise():
self.domain_request_1 = completed_domain_request( self.domain_request_1 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov"

View file

@ -44,6 +44,7 @@ class CsvReportsTest(MockDb):
fake_open = mock_open() fake_open = mock_open()
expected_file_content = [ expected_file_content = [
call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"),
call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"),
call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"),
call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"),
call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"),
@ -65,6 +66,7 @@ class CsvReportsTest(MockDb):
fake_open = mock_open() fake_open = mock_open()
expected_file_content = [ expected_file_content = [
call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"),
call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"),
call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"),
call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"),
call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"),
@ -255,6 +257,7 @@ class ExportDataTest(MockDb, MockEppLib):
"AO email,Security contact email,Status,Expiration date\n" "AO email,Security contact email,Status,Expiration date\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n"
"adomain2.gov,Interstate,(blank),Dns needed\n" "adomain2.gov,Interstate,(blank),Dns needed\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n"
"ddomain3.gov,Federal,Armed Forces Retirement Home,123@mail.gov,On hold,2023-05-25\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,123@mail.gov,On hold,2023-05-25\n"
"defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,(blank),Ready" "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,(blank),Ready"
) )
@ -315,6 +318,7 @@ class ExportDataTest(MockDb, MockEppLib):
"Security contact email,Status\n" "Security contact email,Status\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n"
"adomain2.gov,Interstate,Dns needed\n" "adomain2.gov,Interstate,Dns needed\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n"
"cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n"
"ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n"
) )
@ -365,6 +369,7 @@ class ExportDataTest(MockDb, MockEppLib):
"Domain name,Domain type,Agency,Organization name,City," "Domain name,Domain type,Agency,Organization name,City,"
"State,Security contact email\n" "State,Security contact email\n"
"adomain10.gov,Federal,Armed Forces Retirement Home\n" "adomain10.gov,Federal,Armed Forces Retirement Home\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommission\n"
"cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n"
"ddomain3.gov,Federal,Armed Forces Retirement Home\n" "ddomain3.gov,Federal,Armed Forces Retirement Home\n"
) )
@ -455,6 +460,7 @@ class ExportDataTest(MockDb, MockEppLib):
"State,Status,Expiration date\n" "State,Status,Expiration date\n"
"cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,\n" "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,\n" "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n"
"zdomain9.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" "zdomain9.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n"
"sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n"
"xdomain7.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" "xdomain7.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n"
@ -515,6 +521,7 @@ class ExportDataTest(MockDb, MockEppLib):
"Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n"
"adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n"
"adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n"
"cdomain11.govReadyFederal-ExecutiveWorldWarICentennialCommissionmeoward@rocks.com\n"
"cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,,"
", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n"
"ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n"
@ -551,9 +558,10 @@ class ExportDataTest(MockDb, MockEppLib):
"MANAGED DOMAINS COUNTS AT END DATE\n" "MANAGED DOMAINS COUNTS AT END DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City," "Total,Federal,Interstate,State or territory,Tribal,County,City,"
"Special district,School district,Election office\n" "Special district,School district,Election office\n"
"1,1,0,0,0,0,0,0,0,1\n" "2,2,0,0,0,0,0,0,0,2\n"
"\n" "\n"
"Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n"
"cdomain11.govFederal-Executivemeoward@rocks.com\n"
"cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n"
) )
@ -674,12 +682,12 @@ class HelperFunctions(MockDb):
} }
# Test with distinct # Test with distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True) managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True)
expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 2]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content) self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
# Test without distinct # Test without distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition)
expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1] expected_content = [2, 4, 0, 0, 0, 0, 0, 0, 0, 2]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content) self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
def test_get_sliced_requests(self): def test_get_sliced_requests(self):

View file

@ -467,16 +467,43 @@ def get_sliced_domains(filter_condition, distinct=False):
domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count() domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count()
# Round trip 2: Get counts for other slices # Round trip 2: Get counts for other slices
# This will require either 8 filterd and distinct DB round trips,
# or 2 DB round trips plus iteration on domain_permissions for each domain
if distinct: if distinct:
generic_org_types_query = ( generic_org_types_query = DomainInformation.objects.filter(**filter_condition).values_list(
DomainInformation.objects.filter(**filter_condition).values_list("generic_org_type", flat=True).distinct() "domain_id", "generic_org_type"
) )
# Initialize Counter to store counts for each generic_org_type
generic_org_type_counts = Counter()
# Keep track of domains already counted
domains_counted = set()
# Iterate over distinct domains
for domain_id, generic_org_type in generic_org_types_query:
# Check if the domain has already been counted
if domain_id in domains_counted:
continue
# Get all permissions for the current domain
domain_permissions = DomainInformation.objects.filter(domain_id=domain_id, **filter_condition).values_list(
"domain__permissions", flat=True
)
# Check if the domain has multiple permissions
if len(domain_permissions) > 0:
# Mark the domain as counted
domains_counted.add(domain_id)
# Increment the count for the corresponding generic_org_type
generic_org_type_counts[generic_org_type] += 1
else: else:
generic_org_types_query = DomainInformation.objects.filter(**filter_condition).values_list( generic_org_types_query = DomainInformation.objects.filter(**filter_condition).values_list(
"generic_org_type", flat=True "generic_org_type", flat=True
) )
generic_org_type_counts = Counter(generic_org_types_query) generic_org_type_counts = Counter(generic_org_types_query)
# Extract counts for each generic_org_type
federal = generic_org_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) federal = generic_org_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)
interstate = generic_org_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) interstate = generic_org_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0)
state_or_territory = generic_org_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) state_or_territory = generic_org_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0)
@ -503,21 +530,16 @@ def get_sliced_domains(filter_condition, distinct=False):
] ]
def get_sliced_requests(filter_condition, distinct=False): def get_sliced_requests(filter_condition):
"""Get filtered requests counts sliced by org type and election office.""" """Get filtered requests counts sliced by org type and election office."""
# Round trip 1: Get distinct requests based on filter condition # Round trip 1: Get distinct requests based on filter condition
requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count() requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count()
# Round trip 2: Get counts for other slices # Round trip 2: Get counts for other slices
if distinct: generic_org_types_query = DomainRequest.objects.filter(**filter_condition).values_list(
generic_org_types_query = ( "generic_org_type", flat=True
DomainRequest.objects.filter(**filter_condition).values_list("generic_org_type", flat=True).distinct() )
)
else:
generic_org_types_query = DomainRequest.objects.filter(**filter_condition).values_list(
"generic_org_type", flat=True
)
generic_org_type_counts = Counter(generic_org_types_query) generic_org_type_counts = Counter(generic_org_types_query)
federal = generic_org_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) federal = generic_org_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)