zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-05-30 09:20:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Unit tests

Browse source
This commit is contained in:
Rachid Mrad 2024-06-28 16:34:39 -04:00
parent 59e912b69a
commit b600a26eb8
No known key found for this signature in database
5 changed files with 534 additions and 704 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/registrar/management/commands/generate_current_federal_report.py
View file

@ -50,7 +50,7 @@ class Command(BaseCommand):
# Generate a file locally for upload # Generate a file locally for upload
with open(file_path, "w") as file: with open(file_path, "w") as file:
csv_export.export_data_federal_to_csv(file) csv_export.DomainDataFederal.export_data_to_csv(file)
if check_path and not os.path.exists(file_path): if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")

2
src/registrar/management/commands/generate_current_full_report.py
View file

@ -49,7 +49,7 @@ class Command(BaseCommand):
# Generate a file locally for upload # Generate a file locally for upload
with open(file_path, "w") as file: with open(file_path, "w") as file:
csv_export.export_data_full_to_csv(file) csv_export.DomainDataFull.export_data_to_csv(file)
if check_path and not os.path.exists(file_path): if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")

592
src/registrar/tests/test_reports.py
View file

@ -1,21 +1,23 @@
import csv
import io import io
from django.test import Client, RequestFactory from django.test import Client, RequestFactory
from io import StringIO from io import StringIO
from registrar.models.domain_request import DomainRequest from registrar.models.domain_request import DomainRequest
from registrar.models.domain import Domain from registrar.models.domain import Domain
from registrar.models.utility.generic_helper import convert_queryset_to_dict
from registrar.utility.csv_export import ( from registrar.utility.csv_export import (
export_data_managed_domains_to_csv, DomainDataFull,
export_data_unmanaged_domains_to_csv, DomainDataType,
get_sliced_domains, DomainDataFederal,
get_sliced_requests, DomainGrowth,
write_csv_for_domains, DomainManaged,
DomainUnmanaged,
DomainExport,
DomainRequestExport,
DomainRequestGrowth,
DomainRequestDataFull,
get_default_start_date, get_default_start_date,
get_default_end_date, get_default_end_date,
DomainRequestExport,
) )
from django.db.models import Case, When
from django.core.management import call_command from django.core.management import call_command
from unittest.mock import MagicMock, call, mock_open, patch from unittest.mock import MagicMock, call, mock_open, patch
from api.views import get_current_federal, get_current_full from api.views import get_current_federal, get_current_full
@ -45,10 +47,10 @@ class CsvReportsTest(MockDb):
fake_open = mock_open() fake_open = mock_open()
expected_file_content = [ expected_file_content = [
call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"),
call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"),
call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"),
call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"),
call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"),
] ]
# We don't actually want to write anything for a test case, # We don't actually want to write anything for a test case,
# we just want to verify what is being written. # we just want to verify what is being written.
@ -67,11 +69,12 @@ class CsvReportsTest(MockDb):
fake_open = mock_open() fake_open = mock_open()
expected_file_content = [ expected_file_content = [
call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"),
call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"),
call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"),
call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"),
call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"),
call("adomain2.gov,Interstate,,,,, \r\n"), call("adomain2.gov,Interstate,,,,,\r\n"),
call("zdomain12.gov,Interstate,,,,,\r\n"),
] ]
# We don't actually want to write anything for a test case, # We don't actually want to write anything for a test case,
# we just want to verify what is being written. # we just want to verify what is being written.
@ -202,11 +205,9 @@ class ExportDataTest(MockDb, MockEppLib):
def tearDown(self): def tearDown(self):
super().tearDown() super().tearDown()
def test_export_domains_to_writer_security_emails_and_first_ready(self): @less_console_noise_decorator
"""Test that export_domains_to_writer returns the def test_domain_data_type(self):
expected security email and first_ready value""" """Shows security contacts, domain managers, ao"""
with less_console_noise():
# Add security email information # Add security email information
self.domain_1.name = "defaultsecurity.gov" self.domain_1.name = "defaultsecurity.gov"
self.domain_1.save() self.domain_1.save()
@ -216,48 +217,13 @@ class ExportDataTest(MockDb, MockEppLib):
self.domain_2.security_contact self.domain_2.security_contact
# Invoke setter # Invoke setter
self.domain_3.security_contact self.domain_3.security_contact
# Add a first ready date on the first domain. Leaving the others blank. # Add a first ready date on the first domain. Leaving the others blank.
self.domain_1.first_ready = get_default_start_date() self.domain_1.first_ready = get_default_start_date()
self.domain_1.save() self.domain_1.save()
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Domain type",
"Agency",
"Organization name",
"City",
"State",
"AO",
"AO email",
"Security contact email",
"Status",
"Expiration date",
"First ready on",
]
sort_fields = ["domain__name"]
filter_condition = {
"domain__state__in": [
Domain.State.READY,
Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD,
],
}
# Call the export functions # Call the export functions
write_csv_for_domains( DomainDataType.export_data_to_csv(csv_file)
writer,
columns,
sort_fields,
filter_condition,
should_get_domain_managers=False,
should_write_header=True,
)
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
@ -265,15 +231,26 @@ class ExportDataTest(MockDb, MockEppLib):
# We expect READY domains, # We expect READY domains,
# sorted alphabetially by domain name # sorted alphabetially by domain name
expected_content = ( expected_content = (
"Domain name,Domain type,Agency,Organization name,City,State,AO," "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO,"
"AO email,Security contact email,Status,Expiration date, First ready on\n" "AO email,Security contact email,Domain managers,Invited domain managers\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,Ready,(blank),2024-04-03\n" "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,,"
"adomain2.gov,Interstate,(blank),Dns needed,(blank),(blank)\n" "meoward@rocks.com,\n"
"cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank),2024-04-02\n" "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,"
"ddomain3.gov,Federal,Armed Forces Retirement Home,security@mail.gov,On hold,2023-11-15,(blank)\n" ', ,,dotgov@cisa.dhs.gov,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",'
"defaultsecurity.gov,Federal - Executive,World War I Centennial Commission," "woofwardthethird@rocks.com\n"
"(blank),Ready,(blank),2023-11-01\n" "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,,,"
"zdomain12.govInterstateReady,(blank),2024-04-02\n" "squeaker@rocks.com\n"
"bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,, ,,"
"security@mail.gov,,\n"
"sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n"
"adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,registrar@dotgov.gov,"
"meoward@rocks.com,squeaker@rocks.com\n"
"zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,,meoward@rocks.com,\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
@ -281,50 +258,25 @@ class ExportDataTest(MockDb, MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_write_csv_for_domains(self): @less_console_noise_decorator
"""Test that write_body returns the def test_domain_data_full(self):
existing domain, test that sort by domain name works, """Shows security contacts, filtered by state"""
test that filter works""" # Add security email information
self.domain_1.name = "defaultsecurity.gov"
with less_console_noise(): self.domain_1.save()
# Invoke setter
self.domain_1.security_contact
# Invoke setter
self.domain_2.security_contact
# Invoke setter
self.domain_3.security_contact
# Add a first ready date on the first domain. Leaving the others blank.
self.domain_1.first_ready = get_default_start_date()
self.domain_1.save()
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Domain type",
"Agency",
"Organization name",
"City",
"State",
"AO",
"AO email",
"Submitter",
"Submitter title",
"Submitter email",
"Submitter phone",
"Security contact email",
"Status",
]
sort_fields = ["domain__name"]
filter_condition = {
"domain__state__in": [
Domain.State.READY,
Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD,
],
}
# Call the export functions # Call the export functions
write_csv_for_domains( DomainDataFull.export_data_to_csv(csv_file)
writer,
columns,
sort_fields,
filter_condition,
should_get_domain_managers=False,
should_write_header=True,
)
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
@ -332,15 +284,13 @@ class ExportDataTest(MockDb, MockEppLib):
# We expect READY domains, # We expect READY domains,
# sorted alphabetially by domain name # sorted alphabetially by domain name
expected_content = ( expected_content = (
"Domain name,Domain type,Agency,Organization name,City,State,AO," "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n"
"AO email,Submitter,Submitter title,Submitter email,Submitter phone," "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n"
"Security contact email,Status\n" "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n"
"adomain2.gov,Interstate,Dns needed\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n" "adomain2.gov,Interstate,,,,,registrar@dotgov.gov\n"
"cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" "zdomain12.gov,Interstate,,,,,\n"
"ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n"
"zdomain12.govInterstateReady\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
@ -348,55 +298,37 @@ class ExportDataTest(MockDb, MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_write_domains_body_additional(self): @less_console_noise_decorator
"""An additional test for filters and multi-column sort""" def test_domain_data_federal(self):
"""Shows security contacts, filtered by state and org type"""
with less_console_noise(): # Add security email information
self.domain_1.name = "defaultsecurity.gov"
self.domain_1.save()
# Invoke setter
self.domain_1.security_contact
# Invoke setter
self.domain_2.security_contact
# Invoke setter
self.domain_3.security_contact
# Add a first ready date on the first domain. Leaving the others blank.
self.domain_1.first_ready = get_default_start_date()
self.domain_1.save()
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Domain type",
"Agency",
"Organization name",
"City",
"State",
"Security contact email",
]
sort_fields = ["domain__name", "federal_agency", "generic_org_type"]
filter_condition = {
"generic_org_type__icontains": "federal",
"domain__state__in": [
Domain.State.READY,
Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD,
],
}
# Call the export functions # Call the export functions
write_csv_for_domains( DomainDataFederal.export_data_to_csv(csv_file)
writer,
columns,
sort_fields,
filter_condition,
should_get_domain_managers=False,
should_write_header=True,
)
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
csv_content = csv_file.read() csv_content = csv_file.read()
# We expect READY domains, # We expect READY domains,
# federal only
# sorted alphabetially by domain name # sorted alphabetially by domain name
expected_content = ( expected_content = (
"Domain name,Domain type,Agency,Organization name,City," "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n"
"State,Security contact email\n" "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n"
"adomain10.gov,Federal,Armed Forces Retirement Home\n" "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommission\n" "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n"
"cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n"
"ddomain3.gov,Federal,Armed Forces Retirement Home\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
@ -404,24 +336,10 @@ class ExportDataTest(MockDb, MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): @less_console_noise_decorator
"""Test that domains that are def test_domain_growth(self):
1. READY and their first_ready dates are in range """Shows ready and deleted domains within a date range, sorted"""
2. DELETED and their deleted dates are in range # Remove "Created at" and "First ready" because we can't guess this immutable, dynamically generated test data
are pulled when the growth report conditions are applied to export_domains_to_writed.
Test that ready domains are sorted by first_ready/deleted dates first, names second.
We considered testing export_data_domain_growth_to_csv which calls write_body
and would have been easy to set up, but expected_content would contain created_at dates
which are hard to mock.
TODO: Simplify if created_at is not needed for the report."""
with less_console_noise():
# Create a CSV file in memory
csv_file = StringIO()
writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition
columns = [ columns = [
"Domain name", "Domain name",
"Domain type", "Domain type",
@ -431,164 +349,71 @@ class ExportDataTest(MockDb, MockEppLib):
"State", "State",
"Status", "Status",
"Expiration date", "Expiration date",
# "Created at",
# "First ready",
"Deleted",
] ]
sort_fields = [ sort = {
"created_at", "custom_sort": Case(
"domain__name", When(domain__state=Domain.State.READY, then="domain__created_at"),
] When(domain__state=Domain.State.DELETED, then="domain__deleted"),
sort_fields_for_deleted_domains = [
"domain__deleted",
"domain__name",
]
filter_condition = {
"domain__state__in": [
Domain.State.READY,
],
"domain__first_ready__lte": self.end_date,
"domain__first_ready__gte": self.start_date,
}
filter_conditions_for_deleted_domains = {
"domain__state__in": [
Domain.State.DELETED,
],
"domain__deleted__lte": self.end_date,
"domain__deleted__gte": self.start_date,
}
# Call the export functions
write_csv_for_domains(
writer,
columns,
sort_fields,
filter_condition,
should_get_domain_managers=False,
should_write_header=True,
) )
write_csv_for_domains( }
writer, with patch("registrar.utility.csv_export.DomainGrowth.get_columns", return_value=columns):
columns, with patch("registrar.utility.csv_export.DomainGrowth.get_annotations_for_sort", return_value=sort):
sort_fields_for_deleted_domains, # Create a CSV file in memory
filter_conditions_for_deleted_domains, csv_file = StringIO()
should_get_domain_managers=False, # Call the export functions
should_write_header=False, DomainGrowth.export_data_to_csv(
csv_file,
self.start_date.strftime("%Y-%m-%d"),
self.end_date.strftime("%Y-%m-%d"),
) )
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
csv_content = csv_file.read() csv_content = csv_file.read()
# We expect READY domains first, created between day-2 and day+2, sorted by created_at then name # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name
# and DELETED domains deleted between day-2 and day+2, sorted by deleted then name # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name
expected_content = ( expected_content = (
"Domain name,Domain type,Agency,Organization name,City," "Domain name,Domain type,Agency,Organization name,City,"
"State,Status,Expiration date\n" "State,Status,Expiration date, Deleted\n"
"cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n"
"adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n"
"cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n"
"zdomain12.govInterstateReady(blank)\n" "zdomain12.govInterstateReady(blank)\n"
"zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n"
"sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank)\n" "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n"
"xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank)\n" "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() csv_content = (
csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
)
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_export_domains_to_writer_domain_managers(self): @less_console_noise_decorator
"""Test that export_domains_to_writer returns the def test_domain_managed(self):
expected domain managers. """Shows ready and deleted domains by an end date, sorted
An invited user, woofwardthethird, should also be pulled into this report. An invited user, woofwardthethird, should also be pulled into this report.
squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers).
She should show twice in this report but not in test_export_data_managed_domains_to_csv.""" She should show twice in this report but not in test_DomainManaged."""
with less_console_noise():
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Status",
"Expiration date",
"Domain type",
"Agency",
"Organization name",
"City",
"State",
"AO",
"AO email",
"Security contact email",
]
sort_fields = ["domain__name"]
filter_condition = {
"domain__state__in": [
Domain.State.READY,
Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD,
],
}
# Call the export functions # Call the export functions
write_csv_for_domains( DomainManaged.export_data_to_csv(
writer, csv_file,
columns, self.start_date.strftime("%Y-%m-%d"),
sort_fields, self.end_date.strftime("%Y-%m-%d"),
filter_condition,
should_get_domain_managers=True,
should_write_header=True,
) )
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
csv_content = csv_file.read() csv_content = csv_file.read()
# We expect READY domains,
# sorted alphabetially by domain name
expected_content = (
"Domain name,Status,Expiration date,Domain type,Agency,"
"Organization name,City,State,AO,AO email,"
"Security contact email,Domain manager 1,DM1 status,Domain manager 2,DM2 status,"
"Domain manager 3,DM3 status,Domain manager 4,DM4 status\n"
"adomain10.gov,Ready,(blank),Federal,Armed Forces Retirement Home,,,, , ,squeaker@rocks.com, I\n"
"adomain2.gov,Dns needed,(blank),Interstate,,,,, , , ,meoward@rocks.com, R,squeaker@rocks.com, I\n"
"cdomain11.govReady,(blank),Federal-ExecutiveWorldWarICentennialCommissionmeoward@rocks.comR\n"
"cdomain1.gov,Ready,(blank),Federal - Executive,World War I Centennial Commission,,,"
", , , ,meoward@rocks.com,R,info@example.com,R,big_lebowski@dude.co,R,"
"woofwardthethird@rocks.com,I\n"
"ddomain3.gov,On hold,(blank),Federal,Armed Forces Retirement Home,,,, , , ,,\n"
"zdomain12.gov,Ready,(blank),Interstate,meoward@rocks.com,R\n"
)
# Normalize line endings and remove commas,
# spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_export_data_managed_domains_to_csv(self):
"""Test get counts for domains that have domain managers for two different dates,
get list of managed domains at end_date.
An invited user, woofwardthethird, should also be pulled into this report."""
with less_console_noise():
# Create a CSV file in memory
csv_file = StringIO()
export_data_managed_domains_to_csv(
csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
)
# Reset the CSV file's position to the beginning
csv_file.seek(0)
# Read the content into a variable
csv_content = csv_file.read()
# We expect the READY domain names with the domain managers: Their counts, and listing at end_date. # We expect the READY domain names with the domain managers: Their counts, and listing at end_date.
expected_content = ( expected_content = (
"MANAGED DOMAINS COUNTS AT START DATE\n" "MANAGED DOMAINS COUNTS AT START DATE\n"
@ -601,29 +426,24 @@ class ExportDataTest(MockDb, MockEppLib):
"Special district,School district,Election office\n" "Special district,School district,Election office\n"
"3,2,1,0,0,0,0,0,0,0\n" "3,2,1,0,0,0,0,0,0,0\n"
"\n" "\n"
"Domain name,Domain type,Domain manager 1,DM1 status,Domain manager 2,DM2 status," "Domain name,Domain type,Domain managers,Invited domain managers\n"
"Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" "cdomain11.gov,Federal - Executive,meoward@rocks.com,\n"
"cdomain11.govFederal-Executivemeoward@rocks.com, R\n" 'cdomain1.gov,Federal - Executive,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",'
"cdomain1.gov,Federal - Executive,meoward@rocks.com,R,info@example.com,R," "woofwardthethird@rocks.com\n"
"big_lebowski@dude.co,R,woofwardthethird@rocks.com,I\n" "zdomain12.gov,Interstate,meoward@rocks.com,\n"
"zdomain12.govInterstatemeoward@rocks.com,R\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_export_data_unmanaged_domains_to_csv(self): @less_console_noise_decorator
"""Test get counts for domains that do not have domain managers for two different dates, def test_domain_unmanaged(self):
get list of unmanaged domains at end_date.""" """Shows unmanaged domains by an end date, sorted"""
with less_console_noise():
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
export_data_unmanaged_domains_to_csv( DomainUnmanaged.export_data_to_csv(
csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
) )
@ -655,41 +475,29 @@ class ExportDataTest(MockDb, MockEppLib):
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @less_console_noise_decorator
"""Test that requests that are def test_domain_request_growth(self):
1. SUBMITTED and their submission_date are in range """Shows submitted requests within a date range, sorted"""
are pulled when the growth report conditions are applied to export_requests_to_writed. # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data
Test that requests are sorted by requested domain name. columns = [
""" "Domain request",
"Domain type",
with less_console_noise(): "Federal type",
# "Submitted at",
]
with patch("registrar.utility.csv_export.DomainRequestGrowth.get_columns", return_value=columns):
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file) # Call the export functions
# Define columns, sort fields, and filter condition DomainRequestGrowth.export_data_to_csv(
# We'll skip submission date because it's dynamic and therefore csv_file,
# impossible to set in expected_content self.start_date.strftime("%Y-%m-%d"),
columns = ["Domain request", "Domain type", "Federal type"] self.end_date.strftime("%Y-%m-%d"),
sort_fields = [ )
"requested_domain__name",
]
filter_condition = {
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": self.end_date,
"submission_date__gte": self.start_date,
}
additional_values = ["requested_domain__name"]
all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(all_requests, {}, additional_values)
requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False)
DomainRequestExport.write_csv_for_requests(writer, columns, requests_dict)
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
csv_content = csv_file.read() csv_content = csv_file.read()
# We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
# and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
expected_content = ( expected_content = (
"Domain request,Domain type,Federal type\n" "Domain request,Domain type,Federal type\n"
"city3.gov,Federal,Executive\n" "city3.gov,Federal,Executive\n"
@ -705,69 +513,81 @@ class ExportDataTest(MockDb, MockEppLib):
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
@less_console_noise_decorator @less_console_noise_decorator
def test_full_domain_request_report(self): def test_domain_request_data_full(self):
"""Tests the full domain request report.""" """Tests the full domain request report."""
# Remove "Submitted at" because we can't guess this immutable, dynamically generated test data
columns = [
"Domain request",
# "Submitted at",
"Status",
"Domain type",
"Federal type",
"Federal agency",
"Organization name",
"Election office",
"City",
"State/territory",
"Region",
"Creator first name",
"Creator last name",
"Creator email",
"Creator approved domains count",
"Creator active requests count",
"Alternative domains",
"AO first name",
"AO last name",
"AO email",
"AO title/role",
"Request purpose",
"Request additional details",
"Other contacts",
"CISA regional representative",
"Current websites",
"Investigator",
]
with patch("registrar.utility.csv_export.DomainRequestDataFull.get_columns", return_value=columns):
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file) # Call the export functions
DomainRequestDataFull.export_data_to_csv(csv_file)
# Call the report. Get existing fields from the report itself.
annotations = DomainRequestExport._full_domain_request_annotations()
additional_values = [
"requested_domain__name",
"federal_agency__agency",
"authorizing_official__first_name",
"authorizing_official__last_name",
"authorizing_official__email",
"authorizing_official__title",
"creator__first_name",
"creator__last_name",
"creator__email",
"investigator__email",
]
requests = DomainRequest.objects.exclude(status=DomainRequest.DomainRequestStatus.STARTED)
annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(requests, annotations, additional_values)
requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False)
DomainRequestExport.write_csv_for_requests(writer, DomainRequestExport.all_columns, requests_dict)
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
csv_content = csv_file.read() csv_content = csv_file.read()
print(csv_content) print(csv_content)
self.maxDiff = None
expected_content = ( expected_content = (
# Header # Header
"Domain request,Submitted at,Status,Domain type,Federal type," "Domain request,Status,Domain type,Federal type,"
"Federal agency,Organization name,Election office,City,State/territory," "Federal agency,Organization name,Election office,City,State/territory,"
"Region,Creator first name,Creator last name,Creator email,Creator approved domains count," "Region,Creator first name,Creator last name,Creator email,Creator approved domains count,"
"Creator active requests count,Alternative domains,AO first name,AO last name,AO email," "Creator active requests count,Alternative domains,AO first name,AO last name,AO email,"
"AO title/role,Request purpose,Request additional details,Other contacts," "AO title/role,Request purpose,Request additional details,Other contacts,"
"CISA regional representative,Current websites,Investigator\n" "CISA regional representative,Current websites,Investigator\n"
# Content # Content
"city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com,"
"Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n"
"city3.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"
"cheeseville.gov | city1.gov | igorville.gov,Testy,Tester,testy@town.com,Chief Tester,"
"Purpose of the site,CISA-first-name CISA-last-name | There is more,Meow Tester24 te2@town.com | "
"Testy1232 Tester24 te2@town.com | Testy Tester testy2@town.com,test@igorville.com,"
"city.com | https://www.example2.com | https://www.example.com,\n"
"city4.gov,2024-04-02,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,"
"testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,"
"Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,\n"
"city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com,"
"Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n"
"city6.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,"
"testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," "testy@town.com,"
"Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n"
'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,'
'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name '
"CISA-last-name "
'| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester '
'testy2@town.com"'
',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n'
"city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com,"
"Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester "
"testy2@town.com"
",cisaRep@igorville.gov,city.com,\n"
"city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com,"
"Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester "
"testy2@town.com,"
"cisaRep@igorville.gov,city.com,\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
# spaces and leading/trailing whitespace # spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content) self.assertEqual(csv_content, expected_content)
@ -794,12 +614,12 @@ class HelperFunctions(MockDb):
"domain__first_ready__lte": self.end_date, "domain__first_ready__lte": self.end_date,
} }
# Test with distinct # Test with distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition)
expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content) self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
# Test without distinct # Test without distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition)
expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content) self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
@ -811,6 +631,6 @@ class HelperFunctions(MockDb):
"status": DomainRequest.DomainRequestStatus.SUBMITTED, "status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": self.end_date, "submission_date__lte": self.end_date,
} }
submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition)
expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1]
self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) self.assertEqual(submitted_requests_sliced_at_end_date, expected_content)

186
src/registrar/utility/csv_export.py
View file

@ -15,12 +15,10 @@ from django.db.models import QuerySet, Value, CharField, Count, Q, F
from django.db.models import Case, When, DateField from django.db.models import Case, When, DateField
from django.db.models import ManyToManyField from django.db.models import ManyToManyField
from django.utils import timezone from django.utils import timezone
from django.core.paginator import Paginator
from django.db.models.functions import Concat, Coalesce from django.db.models.functions import Concat, Coalesce
from django.contrib.postgres.aggregates import StringAgg from django.contrib.postgres.aggregates import StringAgg
from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.models.utility.generic_helper import convert_queryset_to_dict
from registrar.templatetags.custom_filters import get_region from registrar.templatetags.custom_filters import get_region
from registrar.utility.enums import DefaultEmail
from registrar.utility.constants import BranchChoices from registrar.utility.constants import BranchChoices
@ -34,14 +32,17 @@ def write_header(writer, columns):
""" """
writer.writerow(columns) writer.writerow(columns)
def get_default_start_date(): def get_default_start_date():
# Default to a date that's prior to our first deployment """Default to a date that's prior to our first deployment"""
return timezone.make_aware(datetime(2023, 11, 1)) return timezone.make_aware(datetime(2023, 11, 1))
def get_default_end_date(): def get_default_end_date():
# Default to now() """Default to now()"""
return timezone.now() return timezone.now()
def format_start_date(start_date): def format_start_date(start_date):
return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date()
@ -49,9 +50,11 @@ def format_start_date(start_date):
def format_end_date(end_date): def format_end_date(end_date):
return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date()
class BaseExport(ABC): class BaseExport(ABC):
""" """
A generic class for exporting data which returns a csv file for the given model. A generic class for exporting data which returns a csv file for the given model.
Base class in an inheritance tree of 3.
""" """
@classmethod @classmethod
@ -207,7 +210,8 @@ class BaseExport(ABC):
related_table_fields = cls.get_related_table_fields() related_table_fields = cls.get_related_table_fields()
model_queryset = ( model_queryset = (
cls.model().objects.select_related(*select_related) cls.model()
.objects.select_related(*select_related)
.prefetch_related(*prefetch_related) .prefetch_related(*prefetch_related)
.filter(filter_conditions) .filter(filter_conditions)
.exclude(exclusions) .exclude(exclusions)
@ -217,7 +221,9 @@ class BaseExport(ABC):
) )
# Convert the queryset to a dictionary (including annotated fields) # Convert the queryset to a dictionary (including annotated fields)
annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) annotated_queryset = cls.annotate_and_retrieve_fields(
model_queryset, computed_fields, related_table_fields, **kwargs
)
models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False)
# Write to csv file before the write_csv # Write to csv file before the write_csv
@ -263,6 +269,7 @@ class BaseExport(ABC):
class DomainExport(BaseExport): class DomainExport(BaseExport):
""" """
A collection of functions which return csv files regarding the Domain model. A collection of functions which return csv files regarding the Domain model.
Second class in an inheritance tree of 3.
""" """
@classmethod @classmethod
@ -279,9 +286,9 @@ class DomainExport(BaseExport):
based on public_contacts, domain_invitations and user_domain_roles based on public_contacts, domain_invitations and user_domain_roles
passed through kwargs. passed through kwargs.
""" """
public_contacts = kwargs.get('public_contacts', {}) public_contacts = kwargs.get("public_contacts", {})
domain_invitations = kwargs.get('domain_invitations', {}) domain_invitations = kwargs.get("domain_invitations", {})
user_domain_roles = kwargs.get('user_domain_roles', {}) user_domain_roles = kwargs.get("user_domain_roles", {})
annotated_domain_infos = [] annotated_domain_infos = []
@ -296,9 +303,11 @@ class DomainExport(BaseExport):
# Annotate with security_contact from public_contacts # Annotate with security_contact from public_contacts
for domain_info in queryset: for domain_info in queryset:
domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) domain_info["security_contact_email"] = public_contacts.get(
domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) domain_info.get("domain__security_contact_registry_id")
domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) )
domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), []))
domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), []))
annotated_domain_infos.append(domain_info) annotated_domain_infos.append(domain_info)
if annotated_domain_infos: if annotated_domain_infos:
@ -316,7 +325,7 @@ class DomainExport(BaseExport):
""" """
Fetch all PublicContact entries and return a mapping of registry_id to email. Fetch all PublicContact entries and return a mapping of registry_id to email.
""" """
public_contacts = PublicContact.objects.values_list('registry_id', 'email') public_contacts = PublicContact.objects.values_list("registry_id", "email")
return {registry_id: email for registry_id, email in public_contacts} return {registry_id: email for registry_id, email in public_contacts}
@classmethod @classmethod
@ -324,7 +333,7 @@ class DomainExport(BaseExport):
""" """
Fetch all DomainInvitation entries and return a mapping of domain to email. Fetch all DomainInvitation entries and return a mapping of domain to email.
""" """
domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email")
return list(domain_invitations) return list(domain_invitations)
@classmethod @classmethod
@ -332,7 +341,7 @@ class DomainExport(BaseExport):
""" """
Fetch all UserDomainRole entries and return a mapping of domain to user__email. Fetch all UserDomainRole entries and return a mapping of domain to user__email.
""" """
user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email")
return list(user_domain_roles) return list(user_domain_roles)
@classmethod @classmethod
@ -360,19 +369,9 @@ class DomainExport(BaseExport):
if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL:
domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}"
if model.get("domain__name") == "18f.gov":
print(f'domain_type {domain_type}')
print(f'federal_agency {model.get("federal_agency")}')
print(f'city {model.get("city")}')
print(f'agency {model.get("agency")}')
print(f'federal_agency__agency {model.get("federal_agency__agency")}')
# create a dictionary of fields which can be included in output. # create a dictionary of fields which can be included in output.
# "extra_fields" are precomputed fields (generated in the DB or parsed). # "extra_fields" are precomputed fields (generated in the DB or parsed).
FIELDS = { FIELDS = {
"Domain name": model.get("domain__name"), "Domain name": model.get("domain__name"),
"Status": human_readable_status, "Status": human_readable_status,
"First ready on": first_ready_on, "First ready on": first_ready_on,
@ -434,6 +433,10 @@ class DomainExport(BaseExport):
class DomainDataType(DomainExport): class DomainDataType(DomainExport):
"""
Shows security contacts, domain managers, ao
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -488,9 +491,9 @@ class DomainDataType(DomainExport):
user_domain_roles = cls.get_all_user_domain_roles() user_domain_roles = cls.get_all_user_domain_roles()
return { return {
'public_contacts': public_contacts, "public_contacts": public_contacts,
'domain_invitations': domain_invitations, "domain_invitations": domain_invitations,
'user_domain_roles': user_domain_roles, "user_domain_roles": user_domain_roles,
} }
@classmethod @classmethod
@ -498,19 +501,14 @@ class DomainDataType(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain", "authorizing_official"]
"domain",
"authorizing_official"
]
@classmethod @classmethod
def get_prefetch_related(cls): def get_prefetch_related(cls):
""" """
Get a list of tables to pass to prefetch_related when building queryset. Get a list of tables to pass to prefetch_related when building queryset.
""" """
return [ return ["permissions"]
"permissions"
]
@classmethod @classmethod
def get_computed_fields(cls, delimiter=", "): def get_computed_fields(cls, delimiter=", "):
@ -545,6 +543,10 @@ class DomainDataType(DomainExport):
class DomainDataFull(DomainExport): class DomainDataFull(DomainExport):
"""
Shows security contacts, filtered by state
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -586,7 +588,7 @@ class DomainDataFull(DomainExport):
public_contacts = cls.get_all_security_emails() public_contacts = cls.get_all_security_emails()
return { return {
'public_contacts': public_contacts, "public_contacts": public_contacts,
} }
@classmethod @classmethod
@ -594,9 +596,7 @@ class DomainDataFull(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain"]
"domain"
]
@classmethod @classmethod
def get_filter_conditions(cls, start_date=None, end_date=None): def get_filter_conditions(cls, start_date=None, end_date=None):
@ -604,7 +604,7 @@ class DomainDataFull(DomainExport):
Get a Q object of filter conditions to filter when building queryset. Get a Q object of filter conditions to filter when building queryset.
""" """
return Q( return Q(
domain__state__in = [ domain__state__in=[
Domain.State.READY, Domain.State.READY,
Domain.State.DNS_NEEDED, Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD, Domain.State.ON_HOLD,
@ -638,6 +638,10 @@ class DomainDataFull(DomainExport):
class DomainDataFederal(DomainExport): class DomainDataFederal(DomainExport):
"""
Shows security contacts, filtered by state and org type
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -679,7 +683,7 @@ class DomainDataFederal(DomainExport):
public_contacts = cls.get_all_security_emails() public_contacts = cls.get_all_security_emails()
return { return {
'public_contacts': public_contacts, "public_contacts": public_contacts,
} }
@classmethod @classmethod
@ -687,9 +691,7 @@ class DomainDataFederal(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain"]
"domain"
]
@classmethod @classmethod
def get_filter_conditions(cls, start_date=None, end_date=None): def get_filter_conditions(cls, start_date=None, end_date=None):
@ -702,7 +704,7 @@ class DomainDataFederal(DomainExport):
Domain.State.READY, Domain.State.READY,
Domain.State.DNS_NEEDED, Domain.State.DNS_NEEDED,
Domain.State.ON_HOLD, Domain.State.ON_HOLD,
] ],
) )
@classmethod @classmethod
@ -732,6 +734,10 @@ class DomainDataFederal(DomainExport):
class DomainGrowth(DomainExport): class DomainGrowth(DomainExport):
"""
Shows ready and deleted domains within a date range, sorted
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -760,10 +766,10 @@ class DomainGrowth(DomainExport):
today = timezone.now().date() today = timezone.now().date()
return { return {
"custom_sort": Case( "custom_sort": Case(
When(domain__state=Domain.State.READY, then='domain__first_ready'), When(domain__state=Domain.State.READY, then="domain__first_ready"),
When(domain__state=Domain.State.DELETED, then='domain__deleted'), When(domain__state=Domain.State.DELETED, then="domain__deleted"),
default=Value(today), # Default value if no conditions match default=Value(today), # Default value if no conditions match
output_field=DateField() output_field=DateField(),
) )
} }
@ -773,9 +779,9 @@ class DomainGrowth(DomainExport):
Returns the sort fields. Returns the sort fields.
""" """
return [ return [
'-domain__state', "-domain__state",
'custom_sort', "custom_sort",
'domain__name', "domain__name",
] ]
@classmethod @classmethod
@ -783,9 +789,7 @@ class DomainGrowth(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain"]
"domain"
]
@classmethod @classmethod
def get_filter_conditions(cls, start_date=None, end_date=None): def get_filter_conditions(cls, start_date=None, end_date=None):
@ -795,12 +799,10 @@ class DomainGrowth(DomainExport):
filter_ready = Q( filter_ready = Q(
domain__state__in=[Domain.State.READY], domain__state__in=[Domain.State.READY],
domain__first_ready__gte=start_date, domain__first_ready__gte=start_date,
domain__first_ready__lte=end_date domain__first_ready__lte=end_date,
) )
filter_deleted = Q( filter_deleted = Q(
domain__state__in=[Domain.State.DELETED], domain__state__in=[Domain.State.DELETED], domain__deleted__gte=start_date, domain__deleted__lte=end_date
domain__deleted__gte=start_date,
domain__deleted__lte=end_date
) )
return filter_ready | filter_deleted return filter_ready | filter_deleted
@ -821,6 +823,10 @@ class DomainGrowth(DomainExport):
class DomainManaged(DomainExport): class DomainManaged(DomainExport):
"""
Shows managed domains by an end date, sorted
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -840,7 +846,7 @@ class DomainManaged(DomainExport):
Returns the sort fields. Returns the sort fields.
""" """
return [ return [
'domain__name', "domain__name",
] ]
@classmethod @classmethod
@ -848,18 +854,14 @@ class DomainManaged(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain"]
"domain"
]
@classmethod @classmethod
def get_prefetch_related(cls): def get_prefetch_related(cls):
""" """
Get a list of tables to pass to prefetch_related when building queryset. Get a list of tables to pass to prefetch_related when building queryset.
""" """
return [ return ["permissions"]
"permissions"
]
@classmethod @classmethod
def get_filter_conditions(cls, start_date=None, end_date=None): def get_filter_conditions(cls, start_date=None, end_date=None):
@ -872,7 +874,6 @@ class DomainManaged(DomainExport):
domain__first_ready__lte=end_date_formatted, domain__first_ready__lte=end_date_formatted,
) )
@classmethod @classmethod
def get_additional_args(cls): def get_additional_args(cls):
""" """
@ -889,8 +890,8 @@ class DomainManaged(DomainExport):
user_domain_roles = cls.get_all_user_domain_roles() user_domain_roles = cls.get_all_user_domain_roles()
return { return {
'domain_invitations': domain_invitations, "domain_invitations": domain_invitations,
'user_domain_roles': user_domain_roles, "user_domain_roles": user_domain_roles,
} }
@classmethod @classmethod
@ -959,6 +960,10 @@ class DomainManaged(DomainExport):
class DomainUnmanaged(DomainExport): class DomainUnmanaged(DomainExport):
"""
Shows unmanaged domains by an end date, sorted
Inherits from BaseExport -> DomainExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -976,7 +981,7 @@ class DomainUnmanaged(DomainExport):
Returns the sort fields. Returns the sort fields.
""" """
return [ return [
'domain__name', "domain__name",
] ]
@classmethod @classmethod
@ -984,18 +989,14 @@ class DomainUnmanaged(DomainExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["domain"]
"domain"
]
@classmethod @classmethod
def get_prefetch_related(cls): def get_prefetch_related(cls):
""" """
Get a list of tables to pass to prefetch_related when building queryset. Get a list of tables to pass to prefetch_related when building queryset.
""" """
return [ return ["permissions"]
"permissions"
]
@classmethod @classmethod
def get_filter_conditions(cls, start_date=None, end_date=None): def get_filter_conditions(cls, start_date=None, end_date=None):
@ -1075,6 +1076,10 @@ class DomainUnmanaged(DomainExport):
class DomainRequestExport(BaseExport): class DomainRequestExport(BaseExport):
"""
A collection of functions which return csv files regarding the DomainRequest model.
Second class in an inheritance tree of 3.
"""
@classmethod @classmethod
def model(cls): def model(cls):
@ -1197,6 +1202,10 @@ class DomainRequestExport(BaseExport):
class DomainRequestGrowth(DomainRequestExport): class DomainRequestGrowth(DomainRequestExport):
"""
Shows submitted requests within a date range, sorted
Inherits from BaseExport -> DomainRequestExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -1238,12 +1247,14 @@ class DomainRequestGrowth(DomainRequestExport):
""" """
Get a list of fields from related tables. Get a list of fields from related tables.
""" """
return [ return ["requested_domain__name"]
"requested_domain__name"
]
class DomainRequestDataFull(DomainRequestExport): class DomainRequestDataFull(DomainRequestExport):
"""
Shows all but STARTED requests
Inherits from BaseExport -> DomainRequestExport
"""
@classmethod @classmethod
def get_columns(cls): def get_columns(cls):
@ -1285,33 +1296,21 @@ class DomainRequestDataFull(DomainRequestExport):
""" """
Get a list of tables to pass to select_related when building queryset. Get a list of tables to pass to select_related when building queryset.
""" """
return [ return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"]
"creator",
"authorizing_official",
"federal_agency",
"investigator",
"requested_domain"
]
@classmethod @classmethod
def get_prefetch_related(cls): def get_prefetch_related(cls):
""" """
Get a list of tables to pass to prefetch_related when building queryset. Get a list of tables to pass to prefetch_related when building queryset.
""" """
return [ return ["current_websites", "other_contacts", "alternative_domains"]
"current_websites",
"other_contacts",
"alternative_domains"
]
@classmethod @classmethod
def get_exclusions(cls): def get_exclusions(cls):
""" """
Get a Q object of exclusion conditions to use when building queryset. Get a Q object of exclusion conditions to use when building queryset.
""" """
return Q( return Q(status__in=[DomainRequest.DomainRequestStatus.STARTED])
status__in=[DomainRequest.DomainRequestStatus.STARTED]
)
@classmethod @classmethod
def get_sort_fields(cls): def get_sort_fields(cls):
@ -1365,7 +1364,6 @@ class DomainRequestDataFull(DomainRequestExport):
"investigator__email", "investigator__email",
] ]
# ============================================================= # # ============================================================= #
# Helper functions for django ORM queries. # # Helper functions for django ORM queries. #
# We are using these rather than pure python for speed reasons. # # We are using these rather than pure python for speed reasons. #

24
src/registrar/views/admin_views.py
View file

@ -49,7 +49,9 @@ class AnalyticsView(View):
"domain__permissions__isnull": False, "domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted, "domain__first_ready__lte": end_date_formatted,
} }
managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(
filter_managed_domains_start_date
)
managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date)
filter_unmanaged_domains_start_date = { filter_unmanaged_domains_start_date = {
@ -60,8 +62,12 @@ class AnalyticsView(View):
"domain__permissions__isnull": True, "domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted, "domain__first_ready__lte": end_date_formatted,
} }
unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(
unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) filter_unmanaged_domains_start_date
)
unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(
filter_unmanaged_domains_end_date
)
filter_ready_domains_start_date = { filter_ready_domains_start_date = {
"domain__state__in": [models.Domain.State.READY], "domain__state__in": [models.Domain.State.READY],
@ -82,7 +88,9 @@ class AnalyticsView(View):
"domain__state__in": [models.Domain.State.DELETED], "domain__state__in": [models.Domain.State.DELETED],
"domain__deleted__lte": end_date_formatted, "domain__deleted__lte": end_date_formatted,
} }
deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(
filter_deleted_domains_start_date
)
deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date)
filter_requests_start_date = { filter_requests_start_date = {
@ -102,8 +110,12 @@ class AnalyticsView(View):
"status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "status": models.DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": end_date_formatted, "submission_date__lte": end_date_formatted,
} }
submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(
submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) filter_submitted_requests_start_date
)
submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(
filter_submitted_requests_end_date
)
context = dict( context = dict(
# Generate a dictionary of context variables that are common across all admin templates # Generate a dictionary of context variables that are common across all admin templates