zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-05-16 09:37:03 +02:00
Code Issues Projects Releases Packages Wiki Activity

WIP unit testing

Browse source
This commit is contained in:
Rachid Mrad 2024-03-04 17:24:53 -05:00
parent 717f71f038
commit c89d76fb47
No known key found for this signature in database
9 changed files with 547 additions and 241 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/registrar/admin.py
View file

@ -374,8 +374,8 @@ def analytics(request):
avg_approval_time = last_30_days_approved_applications.annotate(
approval_time=F("approved_domain__created_at") - F("submission_date")
).aggregate(Avg("approval_time"))["approval_time__avg"]
# format the timedelta?
avg_approval_time = str(avg_approval_time)
# Format the timedelta to display only days
avg_approval_time = f"{avg_approval_time.days} days"
start_date = request.GET.get("start_date", "")
end_date = request.GET.get("end_date", "")
@ -383,75 +383,69 @@ def analytics(request):
start_date_formatted = csv_export.format_start_date(start_date)
end_date_formatted = csv_export.format_end_date(end_date)
# Managed vs Unmanaged
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": start_date_formatted,
}
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted,
}
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date)
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date)
filter_unmanaged_domains_start_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": start_date_formatted,
}
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date)
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted,
}
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date)
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted,
}
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date)
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date)
# Ready and Deleted domains
filter_ready_domains_start_date = {
"domain__state__in": [Domain.State.READY],
"domain__first_ready__lte": start_date_formatted,
}
filter_ready_domains_end_date = {
"domain__state__in": [Domain.State.READY],
"domain__first_ready__lte": end_date_formatted,
}
ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date)
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
filter_deleted_domains_start_date = {
"domain__state__in": [Domain.State.DELETED],
"domain__deleted__lte": start_date_formatted,
}
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
filter_ready_domains_end_date = {
"domain__state__in": [Domain.State.READY],
"domain__first_ready__lte": end_date_formatted,
}
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
filter_deleted_domains_end_date = {
"domain__state__in": [Domain.State.DELETED],
"domain__deleted__lte": end_date_formatted,
}
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date)
# Created and Submitted requests
filter_requests_start_date = {
"created_at__lte": start_date_formatted,
}
filter_requests_end_date = {
"created_at__lte": end_date_formatted,
}
requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date)
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
filter_submitted_requests_start_date = {
"status": DomainApplication.ApplicationStatus.SUBMITTED,
"submission_date__lte": start_date_formatted,
}
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
filter_requests_end_date = {
"created_at__lte": end_date_formatted,
}
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
filter_submitted_requests_end_date = {
"status": DomainApplication.ApplicationStatus.SUBMITTED,
"submission_date__lte": end_date_formatted,
}
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date)
context = dict(
@ -459,6 +453,7 @@ def analytics(request):
data=dict(
user_count=models.User.objects.all().count(),
domain_count=models.Domain.objects.all().count(),
ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(),
last_30_days_applications=last_30_days_applications.count(),
last_30_days_approved_applications=last_30_days_approved_applications.count(),
average_application_approval_time_last_30_days=avg_approval_time,
@ -1096,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin):
search_help_text = "Search by domain or submitter."
fieldsets = [
(None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}),
(None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}),
(
"Type of organization",
{
@ -1448,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin):
search_fields = ["name"]
search_help_text = "Search by domain name."
change_form_template = "django/admin/domain_change_form.html"
readonly_fields = ["state", "expiration_date", "first_ready", "deleted"]
readonly_fields = ["state", "expiration_date", "deleted"]
# Table ordering
ordering = ["name"]

3
src/registrar/assets/sass/_theme/_admin.scss
View file

@ -319,6 +319,9 @@ input.admin-confirm-button {
.usa-icon {
top: 2px;
}
a.button:active, a.button:focus {
text-decoration: none;
}
}
.module--custom {

2
src/registrar/models/domain.py
View file

@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper):
first_ready = DateField(
null=True,
editable=False,
editable=True,
help_text="The last time this domain moved into the READY state",
)

17
src/registrar/templates/admin/analytics.html
View file

@ -16,6 +16,7 @@
<ul>
<li>User Count: {{ data.user_count }}</li>
<li>Domain Count: {{ data.domain_count }}</li>
<li>Domains in READY state: {{ data.ready_domain_count }}</li>
<li>Domain applications (last 30 days): {{ data.last_30_days_applications }}</li>
<li>Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}</li>
<li>Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}</li>
@ -63,8 +64,6 @@
{% comment %}
Inputs of type date suck for accessibility.
We'll need to replace those guys with a django form once we figure out how to hook one onto this page.
The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/
See the commit "Review for ticket #999"
{% endcomment %}
<div class="display-flex flex-align-baseline margin-top-1 margin-bottom-2">
@ -107,7 +106,7 @@
</button>
</li>
<li class="usa-button-group__item">
<button class="button exportLink" data-export-url="{% url 'analytics' %}" type="button">
<button class="button exportLink usa-button--accent-cool" data-export-url="{% url 'analytics' %}" type="button">
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
<use xlink:href="{%static 'img/sprite.svg'%}#assessment"></use>
</svg><span class="margin-left-05">Update charts</span>
@ -116,18 +115,18 @@
</ul>
<div class="grid-row grid-gap-2 margin-y-2">
<div class="grid-col">
<canvas id="myChart" width="400" height="200"
data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}"
data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}"
></canvas>
</div>
<div class="grid-col">
<canvas id="myChart2" width="400" height="200"
data-list-one="{{data.managed_domains_sliced_at_start_date}}"
data-list-two="{{data.managed_domains_sliced_at_end_date}}"
></canvas>
</div>
<div class="grid-col">
<canvas id="myChart" width="400" height="200"
data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}"
data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}"
></canvas>
</div>
</div>
<div class="grid-row grid-gap-2 margin-y-2">

2
src/registrar/templates/admin/app_list.html
View file

@ -71,5 +71,5 @@
<div class="module module--custom">
<h2>Analytics</h2>
<a class="display-block padding-y-1 padding-x-2" href="{% url 'analytics' %}">Dashboard</a>
<a class="display-block padding-y-1 padding-x-1" href="{% url 'analytics' %}">Dashboard</a>
</div>

232
src/registrar/tests/data/mocks.py Normal file
View file

@ -0,0 +1,232 @@
from django.test import TestCase
from django.contrib.auth import get_user_model
from registrar.models.domain_application import DomainApplication
from registrar.models.domain_information import DomainInformation
from registrar.models.domain import Domain
from registrar.models.user_domain_role import UserDomainRole
from registrar.models.public_contact import PublicContact
from registrar.models.user import User
from datetime import date, datetime, timedelta
from django.utils import timezone
from registrar.tests.common import MockEppLib
class MockDb(MockEppLib):
def setUp(self):
super().setUp()
username = "test_user"
first_name = "First"
last_name = "Last"
email = "info@example.com"
self.user = get_user_model().objects.create(
username=username, first_name=first_name, last_name=last_name, email=email
)
self.domain_1, _ = Domain.objects.get_or_create(
name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now()
)
self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED)
self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD)
self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN)
self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN)
self.domain_5, _ = Domain.objects.get_or_create(
name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1))
)
self.domain_6, _ = Domain.objects.get_or_create(
name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16))
)
self.domain_7, _ = Domain.objects.get_or_create(
name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now()
)
self.domain_8, _ = Domain.objects.get_or_create(
name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now()
)
# We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today())
# and a specific time (using datetime.min.time()).
# Deleted yesterday
self.domain_9, _ = Domain.objects.get_or_create(
name="zdomain9.gov",
state=Domain.State.DELETED,
deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())),
)
# ready tomorrow
self.domain_10, _ = Domain.objects.get_or_create(
name="adomain10.gov",
state=Domain.State.READY,
first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())),
)
self.domain_information_1, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_1,
organization_type="federal",
federal_agency="World War I Centennial Commission",
federal_type="executive",
is_election_board=True
)
self.domain_information_2, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_2,
organization_type="interstate",
is_election_board=True
)
self.domain_information_3, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_3,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=True
)
self.domain_information_4, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_4,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=True
)
self.domain_information_5, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_5,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
self.domain_information_6, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_6,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
self.domain_information_7, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_7,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
self.domain_information_8, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_8,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
self.domain_information_9, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_9,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
self.domain_information_10, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_10,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
is_election_board=False
)
meoward_user = get_user_model().objects.create(
username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com"
)
lebowski_user = get_user_model().objects.create(
username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co"
)
# Test for more than 1 domain manager
_, created = UserDomainRole.objects.get_or_create(
user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER
)
_, created = UserDomainRole.objects.get_or_create(
user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER
)
_, created = UserDomainRole.objects.get_or_create(
user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER
)
# Test for just 1 domain manager
_, created = UserDomainRole.objects.get_or_create(
user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER
)
# self.domain_request_1, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# requested_domain=self.domain_1.name,
# organization_type="federal",
# federal_agency="World War I Centennial Commission",
# federal_type="executive",
# is_election_board=True
# )
# self.domain_request_2, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_2,
# organization_type="interstate",
# is_election_board=True
# )
# self.domain_request_3, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_3,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=True
# )
# self.domain_request_4, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_4,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=True
# )
# self.domain_request_5, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_5,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
# self.domain_request_6, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_6,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
# self.domain_request_7, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_7,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
# self.domain_request_8, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_8,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
# self.domain_information_9, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_9,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
# self.domain_information_10, _ = DomainApplication.objects.get_or_create(
# creator=self.user,
# domain=self.domain_10,
# organization_type="federal",
# federal_agency="Armed Forces Retirement Home",
# is_election_board=False
# )
def tearDown(self):
PublicContact.objects.all().delete()
Domain.objects.all().delete()
DomainInformation.objects.all().delete()
User.objects.all().delete()
UserDomainRole.objects.all().delete()
super().tearDown()

4
src/registrar/tests/test_admin_views.py
View file

@ -3,7 +3,7 @@ from django.urls import reverse
from registrar.tests.common import create_superuser
class TestViews(TestCase):
class TestAdminViews(TestCase):
def setUp(self):
self.client = Client(HTTP_HOST="localhost:8080")
self.superuser = create_superuser()
@ -26,7 +26,7 @@ class TestViews(TestCase):
# Construct the URL for the export data view with start_date and end_date parameters:
# This stuff is currently done in JS
export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}"
export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}"
# Make a GET request to the export data page
response = self.client.get(export_data_url)

370
src/registrar/tests/test_reports.py
View file

@ -8,9 +8,12 @@ from registrar.models.public_contact import PublicContact
from registrar.models.user import User
from django.contrib.auth import get_user_model
from registrar.models.user_domain_role import UserDomainRole
from registrar.tests.common import MockEppLib
from registrar.tests.data.mocks import MockDb
from registrar.utility.csv_export import (
write_csv,
format_end_date,
format_start_date,
get_sliced_domains,
write_domains_csv,
get_default_start_date,
get_default_end_date,
)
@ -231,136 +234,11 @@ class CsvReportsTest(TestCase):
self.assertEqual(expected_file_content, response.content)
class ExportDataTest(MockEppLib):
class ExportDataTest(MockDb):
def setUp(self):
super().setUp()
username = "test_user"
first_name = "First"
last_name = "Last"
email = "info@example.com"
self.user = get_user_model().objects.create(
username=username, first_name=first_name, last_name=last_name, email=email
)
self.domain_1, _ = Domain.objects.get_or_create(
name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now()
)
self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED)
self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD)
self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN)
self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN)
self.domain_5, _ = Domain.objects.get_or_create(
name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1))
)
self.domain_6, _ = Domain.objects.get_or_create(
name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16))
)
self.domain_7, _ = Domain.objects.get_or_create(
name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now()
)
self.domain_8, _ = Domain.objects.get_or_create(
name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now()
)
# We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today())
# and a specific time (using datetime.min.time()).
# Deleted yesterday
self.domain_9, _ = Domain.objects.get_or_create(
name="zdomain9.gov",
state=Domain.State.DELETED,
deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())),
)
# ready tomorrow
self.domain_10, _ = Domain.objects.get_or_create(
name="adomain10.gov",
state=Domain.State.READY,
first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())),
)
self.domain_information_1, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_1,
organization_type="federal",
federal_agency="World War I Centennial Commission",
federal_type="executive",
)
self.domain_information_2, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_2,
organization_type="interstate",
)
self.domain_information_3, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_3,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_4, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_4,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_5, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_5,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_6, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_6,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_7, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_7,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_8, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_8,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_9, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_9,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
self.domain_information_10, _ = DomainInformation.objects.get_or_create(
creator=self.user,
domain=self.domain_10,
organization_type="federal",
federal_agency="Armed Forces Retirement Home",
)
meoward_user = get_user_model().objects.create(
username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com"
)
# Test for more than 1 domain manager
_, created = UserDomainRole.objects.get_or_create(
user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER
)
_, created = UserDomainRole.objects.get_or_create(
user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER
)
# Test for just 1 domain manager
_, created = UserDomainRole.objects.get_or_create(
user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER
)
def tearDown(self):
PublicContact.objects.all().delete()
Domain.objects.all().delete()
DomainInformation.objects.all().delete()
User.objects.all().delete()
UserDomainRole.objects.all().delete()
super().tearDown()
def test_export_domains_to_writer_security_emails(self):
@ -403,7 +281,7 @@ class ExportDataTest(MockEppLib):
}
self.maxDiff = None
# Call the export functions
write_csv(
write_domains_csv(
writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True
)
@ -427,7 +305,7 @@ class ExportDataTest(MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_write_csv(self):
def test_write_domains_csv(self):
"""Test that write_body returns the
existing domain, test that sort by domain name works,
test that filter works"""
@ -462,7 +340,7 @@ class ExportDataTest(MockEppLib):
],
}
# Call the export functions
write_csv(
write_domains_csv(
writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True
)
# Reset the CSV file's position to the beginning
@ -486,7 +364,7 @@ class ExportDataTest(MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_write_body_additional(self):
def test_write_domains_body_additional(self):
"""An additional test for filters and multi-column sort"""
with less_console_noise():
# Create a CSV file in memory
@ -512,7 +390,7 @@ class ExportDataTest(MockEppLib):
],
}
# Call the export functions
write_csv(
write_domains_csv(
writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True
)
# Reset the CSV file's position to the beginning
@ -535,7 +413,7 @@ class ExportDataTest(MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_write_body_with_date_filter_pulls_domains_in_range(self):
def test_write_domains_body_with_date_filter_pulls_domains_in_range(self):
"""Test that domains that are
1. READY and their first_ready dates are in range
2. DELETED and their deleted dates are in range
@ -546,7 +424,7 @@ class ExportDataTest(MockEppLib):
and would have been easy to set up, but expected_content would contain created_at dates
which are hard to mock.
TODO: Simplify is created_at is not needed for the report."""
TODO: Simplify if created_at is not needed for the report."""
with less_console_noise():
# Create a CSV file in memory
csv_file = StringIO()
@ -591,7 +469,7 @@ class ExportDataTest(MockEppLib):
}
# Call the export functions
write_csv(
write_domains_csv(
writer,
columns,
sort_fields,
@ -599,7 +477,7 @@ class ExportDataTest(MockEppLib):
get_domain_managers=False,
should_write_header=True,
)
write_csv(
write_domains_csv(
writer,
columns,
sort_fields_for_deleted_domains,
@ -664,7 +542,7 @@ class ExportDataTest(MockEppLib):
}
self.maxDiff = None
# Call the export functions
write_csv(
write_domains_csv(
writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True
)
@ -677,11 +555,11 @@ class ExportDataTest(MockEppLib):
expected_content = (
"Domain name,Status,Expiration date,Domain type,Agency,"
"Organization name,City,State,AO,AO email,"
"Security contact email,Domain manager email 1,Domain manager email 2,\n"
"Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n"
"adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n"
"adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n"
"cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,,"
", , , ,meoward@rocks.com,info@example.com\n"
", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n"
"ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n"
)
# Normalize line endings and remove commas,
@ -690,6 +568,210 @@ class ExportDataTest(MockEppLib):
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_export_data_managed_domains_to_csv(self):
""""""
with less_console_noise():
# Create a CSV file in memory
csv_file = StringIO()
writer = csv.writer(csv_file)
end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time()))
start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time()))
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Domain type",
]
sort_fields = [
"domain__name",
]
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": start_date,
}
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
# Call the export functions
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(managed_domains_sliced_at_start_date)
writer.writerow([])
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date,
}
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date)
writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(managed_domains_sliced_at_end_date)
writer.writerow([])
write_domains_csv(
writer,
columns,
sort_fields,
filter_managed_domains_end_date,
get_domain_managers=True,
should_write_header=True,
)
# Reset the CSV file's position to the beginning
csv_file.seek(0)
# Read the content into a variable
csv_content = csv_file.read()
self.maxDiff=None
# We expect the READY domain names with the domain managers: Their counts, and listing at end_date.
expected_content = (
"MANAGED DOMAINS COUNTS AT START DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n"
"0,0,0,0,0,0,0,0,0,0\n"
"\n"
"MANAGED DOMAINS COUNTS AT END DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n"
"1,1,0,0,0,0,0,0,0,1\n"
"\n"
"Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n"
"cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n"
)
# Normalize line endings and remove commas,
# spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_export_data_unmanaged_domains_to_csv(self):
""""""
with less_console_noise():
# Create a CSV file in memory
csv_file = StringIO()
writer = csv.writer(csv_file)
end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time()))
start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time()))
# Define columns, sort fields, and filter condition
columns = [
"Domain name",
"Domain type",
]
sort_fields = [
"domain__name",
]
filter_unmanaged_domains_start_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": start_date,
}
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date)
# Call the export functions
writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(unmanaged_domains_sliced_at_start_date)
writer.writerow([])
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": end_date,
}
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date)
writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(unmanaged_domains_sliced_at_end_date)
writer.writerow([])
write_domains_csv(
writer,
columns,
sort_fields,
filter_unmanaged_domains_end_date,
get_domain_managers=False,
should_write_header=True,
)
# Reset the CSV file's position to the beginning
csv_file.seek(0)
# Read the content into a variable
csv_content = csv_file.read()
self.maxDiff=None
# We expect the READY domain names with the domain managers: Their counts, and listing at end_date.
expected_content = (
"UNMANAGED DOMAINS COUNTS AT START DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n"
"0,0,0,0,0,0,0,0,0,0\n"
"\n"
"UNMANAGED DOMAINS COUNTS AT END DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n"
"1,1,0,0,0,0,0,0,0,0\n"
"\n"
"Domain name,Domain type\n"
"adomain10.gov,Federal\n"
)
# Normalize line endings and remove commas,
# spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
def test_write_requests_body_with_date_filter_pulls_requests_in_range(self):
"""Test that requests that are
1. SUBMITTED and their submission_date are in range
are pulled when the growth report conditions are applied to export_requests_to_writed.
Test that requests are sorted by requested domain name.
"""
pass
class HelperFunctions(TestCase):
"""This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy."""
@ -704,3 +786,11 @@ class HelperFunctions(TestCase):
expected_date = timezone.now()
actual_date = get_default_end_date()
self.assertEqual(actual_date.date(), expected_date.date())
def get_sliced_domains(self):
"""Should get fitered domains counts sliced by org type and election office."""
pass
def test_get_sliced_requests(self):
"""Should get fitered requests counts sliced by org type and election office."""
pass

109
src/registrar/utility/csv_export.py
View file

@ -25,9 +25,10 @@ def write_header(writer, columns):
def get_domain_infos(filter_condition, sort_fields):
domain_infos = (
DomainInformation.objects.select_related("domain", "authorizing_official")
DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions")
.filter(**filter_condition)
.order_by(*sort_fields)
.distinct()
)
# Do a mass concat of the first and last name fields for authorizing_official.
@ -44,7 +45,7 @@ def get_domain_infos(filter_condition, sort_fields):
return domain_infos_cleaned
def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False):
def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False):
"""Given a set of columns, generate a new row from cleaned column data"""
# Domain should never be none when parsing this information
@ -136,7 +137,7 @@ def update_columns_with_domain_managers(columns, max_dm_count):
columns.append(f"Domain manager email {i}")
def write_csv(
def write_domains_csv(
writer,
columns,
sort_fields,
@ -145,8 +146,8 @@ def write_csv(
should_write_header=True,
):
"""
Receives params from the parent methods and outputs a CSV with fltered and sorted domains.
Works with write_header as longas the same writer object is passed.
Receives params from the parent methods and outputs a CSV with filtered and sorted domains.
Works with write_header as long as the same writer object is passed.
get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv
should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice
"""
@ -172,7 +173,7 @@ def write_csv(
rows = []
for domain_info in page.object_list:
try:
row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers)
row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers)
rows.append(row)
except ValueError:
# This should not happen. If it does, just skip this row.
@ -188,7 +189,6 @@ def write_csv(
def get_requests(filter_condition, sort_fields):
requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields)
return requests
@ -235,7 +235,8 @@ def write_requests_csv(
filter_condition,
should_write_header=True,
):
""" """
"""Receives params from the parent methods and outputs a CSV with filtered and sorted requests.
Works with write_header as long as the same writer object is passed."""
all_requetsts = get_requests(filter_condition, sort_fields)
@ -295,7 +296,7 @@ def export_data_type_to_csv(csv_file):
Domain.State.ON_HOLD,
],
}
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True)
write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True)
def export_data_full_to_csv(csv_file):
@ -326,7 +327,7 @@ def export_data_full_to_csv(csv_file):
Domain.State.ON_HOLD,
],
}
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
def export_data_federal_to_csv(csv_file):
@ -358,7 +359,7 @@ def export_data_federal_to_csv(csv_file):
Domain.State.ON_HOLD,
],
}
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
def get_default_start_date():
@ -426,8 +427,8 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date):
"domain__deleted__gte": start_date_formatted,
}
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
write_csv(
write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
write_domains_csv(
writer,
columns,
sort_fields_for_deleted_domains,
@ -440,19 +441,19 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date):
def get_sliced_domains(filter_condition):
"""Get fitered domains counts sliced by org type and election office."""
domains = DomainInformation.objects.all().filter(**filter_condition)
domains = DomainInformation.objects.all().filter(**filter_condition).distinct()
domains_count = domains.count()
federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count()
federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count()
interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count()
state_or_territory = domains.filter(
organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY
).count()
tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count()
county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count()
city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count()
special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count()
school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count()
election_board = domains.filter(is_election_board=True).count()
).distinct().count()
tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count()
county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count()
city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count()
special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
election_board = domains.filter(is_election_board=True).distinct().count()
return [
domains_count,
@ -471,19 +472,19 @@ def get_sliced_domains(filter_condition):
def get_sliced_requests(filter_condition):
"""Get fitered requests counts sliced by org type and election office."""
requests = DomainApplication.objects.all().filter(**filter_condition)
requests = DomainApplication.objects.all().filter(**filter_condition).distinct()
requests_count = requests.count()
federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count()
interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count()
federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count()
interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count()
state_or_territory = requests.filter(
organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY
).count()
tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count()
county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count()
city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count()
special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count()
school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count()
election_board = requests.filter(is_election_board=True).count()
).distinct().count()
tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count()
county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count()
city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count()
special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
election_board = requests.filter(is_election_board=True).distinct().count()
return [
requests_count,
@ -500,7 +501,8 @@ def get_sliced_requests(filter_condition):
def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
"""Get domains have domain managers for two different dates."""
"""Get counts for domains that have domain managers for two different dates,
get list of domains at end_date."""
start_date_formatted = format_start_date(start_date)
end_date_formatted = format_end_date(end_date)
@ -512,14 +514,13 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
sort_fields = [
"domain__name",
]
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": start_date_formatted,
}
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"])
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow(
[
"Total",
@ -537,16 +538,6 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
writer.writerow(managed_domains_sliced_at_start_date)
writer.writerow([])
write_csv(
writer,
columns,
sort_fields,
filter_managed_domains_start_date,
get_domain_managers=True,
should_write_header=True,
)
writer.writerow([])
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted,
@ -571,7 +562,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
writer.writerow(managed_domains_sliced_at_end_date)
writer.writerow([])
write_csv(
write_domains_csv(
writer,
columns,
sort_fields,
@ -582,7 +573,8 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
"""Get domains that do not have domain managers for two different dates."""
"""Get counts for domains that do not have domain managers for two different dates,
get list of domains at end_date."""
start_date_formatted = format_start_date(start_date)
end_date_formatted = format_end_date(end_date)
@ -619,16 +611,6 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
writer.writerow(unmanaged_domains_sliced_at_start_date)
writer.writerow([])
write_csv(
writer,
columns,
sort_fields,
filter_unmanaged_domains_start_date,
get_domain_managers=True,
should_write_header=True,
)
writer.writerow([])
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted,
@ -653,18 +635,23 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
writer.writerow(unmanaged_domains_sliced_at_end_date)
writer.writerow([])
write_csv(
write_domains_csv(
writer,
columns,
sort_fields,
filter_unmanaged_domains_end_date,
get_domain_managers=True,
get_domain_managers=False,
should_write_header=True,
)
def export_data_requests_growth_to_csv(csv_file, start_date, end_date):
""" """
"""
Growth report:
Receive start and end dates from the view, parse them.
Request from write_requests_body SUBMITTED requests that are created between
the start and end dates. Specify sort params.
"""
start_date_formatted = format_start_date(start_date)
end_date_formatted = format_end_date(end_date)
@ -676,7 +663,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date):
"Submission date",
]
sort_fields = [
# "domain__name",
"requested_domain__name",
]
filter_condition = {
"status": DomainApplication.ApplicationStatus.SUBMITTED,