Refactors, most notably reduce DB trips in get_sliced methods

This commit is contained in:
Rachid Mrad 2024-03-14 18:29:06 -04:00
parent 8147b5f812
commit 427e110d22
No known key found for this signature in database
9 changed files with 239 additions and 315 deletions

View file

@ -1,13 +1,12 @@
from datetime import date from datetime import date
import logging import logging
import datetime
import copy import copy
from django import forms from django import forms
from django.db.models import Avg, F, Value, CharField, Q from django.db.models import Value, CharField, Q
from django.db.models.functions import Concat, Coalesce from django.db.models.functions import Concat, Coalesce
from django.http import HttpResponseRedirect from django.http import HttpResponseRedirect
from django.shortcuts import redirect, render from django.shortcuts import redirect
from django_fsm import get_available_FIELD_transitions from django_fsm import get_available_FIELD_transitions
from django.contrib import admin, messages from django.contrib import admin, messages
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
@ -17,7 +16,6 @@ from django.urls import reverse
from dateutil.relativedelta import relativedelta # type: ignore from dateutil.relativedelta import relativedelta # type: ignore
from epplibwrapper.errors import ErrorCode, RegistryError from epplibwrapper.errors import ErrorCode, RegistryError
from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website
from registrar.utility import csv_export
from registrar.utility.errors import FSMApplicationError, FSMErrorCodes from registrar.utility.errors import FSMApplicationError, FSMErrorCodes
from registrar.views.utility.mixins import OrderableFieldsMixin from registrar.views.utility.mixins import OrderableFieldsMixin
from django.contrib.admin.views.main import ORDER_VAR from django.contrib.admin.views.main import ORDER_VAR
@ -437,123 +435,6 @@ class UserContactInline(admin.StackedInline):
model = models.Contact model = models.Contact
def analytics(request):
"""View for the reports page."""
thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30)
last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago)
last_30_days_approved_applications = models.DomainRequest.objects.filter(
created_at__gt=thirty_days_ago, status=DomainRequest.DomainRequestStatus.APPROVED
)
avg_approval_time = last_30_days_approved_applications.annotate(
approval_time=F("approved_domain__created_at") - F("submission_date")
).aggregate(Avg("approval_time"))["approval_time__avg"]
# Format the timedelta to display only days
if avg_approval_time is not None:
avg_approval_time = f"{avg_approval_time.days} days"
else:
avg_approval_time = "No approvals to use"
# The start and end dates are passed as url params
start_date = request.GET.get("start_date", "")
end_date = request.GET.get("end_date", "")
start_date_formatted = csv_export.format_start_date(start_date)
end_date_formatted = csv_export.format_end_date(end_date)
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": start_date_formatted,
}
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted,
}
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date)
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date)
filter_unmanaged_domains_start_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": start_date_formatted,
}
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted,
}
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date)
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date)
filter_ready_domains_start_date = {
"domain__state__in": [Domain.State.READY],
"domain__first_ready__lte": start_date_formatted,
}
filter_ready_domains_end_date = {
"domain__state__in": [Domain.State.READY],
"domain__first_ready__lte": end_date_formatted,
}
ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date)
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
filter_deleted_domains_start_date = {
"domain__state__in": [Domain.State.DELETED],
"domain__deleted__lte": start_date_formatted,
}
filter_deleted_domains_end_date = {
"domain__state__in": [Domain.State.DELETED],
"domain__deleted__lte": end_date_formatted,
}
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date)
filter_requests_start_date = {
"created_at__lte": start_date_formatted,
}
filter_requests_end_date = {
"created_at__lte": end_date_formatted,
}
requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date)
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
filter_submitted_requests_start_date = {
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": start_date_formatted,
}
filter_submitted_requests_end_date = {
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": end_date_formatted,
}
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date)
context = dict(
**admin.site.each_context(request),
data=dict(
user_count=models.User.objects.all().count(),
domain_count=models.Domain.objects.all().count(),
ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(),
last_30_days_applications=last_30_days_applications.count(),
last_30_days_approved_applications=last_30_days_approved_applications.count(),
average_application_approval_time_last_30_days=avg_approval_time,
managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date,
unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date,
managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date,
unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date,
ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date,
deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date,
ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date,
deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date,
requests_sliced_at_start_date=requests_sliced_at_start_date,
submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date,
requests_sliced_at_end_date=requests_sliced_at_end_date,
submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date,
start_date=start_date,
end_date=end_date,
),
)
return render(request, "admin/analytics.html", context)
class MyUserAdmin(BaseUserAdmin): class MyUserAdmin(BaseUserAdmin):
"""Custom user admin class to use our inlines.""" """Custom user admin class to use our inlines."""

View file

@ -330,7 +330,8 @@ CSP_FORM_ACTION = allowed_sources
# Google analytics requires that we relax our otherwise # Google analytics requires that we relax our otherwise
# strict CSP by allowing scripts to run from their domain # strict CSP by allowing scripts to run from their domain
# and inline with a nonce, as well as allowing connections back to their domain # and inline with a nonce, as well as allowing connections back to their domain.
# Note: If needed, we can embed chart.js instead of using the CDN
CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"]
CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"]
CSP_INCLUDE_NONCE_IN = ["script-src-elem"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"]

View file

@ -9,7 +9,6 @@ from django.urls import include, path
from django.views.generic import RedirectView from django.views.generic import RedirectView
from registrar import views from registrar import views
from registrar.admin import analytics
from registrar.views.admin_views import ( from registrar.views.admin_views import (
ExportDataDomainsGrowth, ExportDataDomainsGrowth,
ExportDataFederal, ExportDataFederal,
@ -18,6 +17,7 @@ from registrar.views.admin_views import (
ExportDataRequestsGrowth, ExportDataRequestsGrowth,
ExportDataType, ExportDataType,
ExportDataUnmanagedDomains, ExportDataUnmanagedDomains,
AnalyticsView,
) )
from registrar.views.domain_request import Step from registrar.views.domain_request import Step
@ -96,7 +96,7 @@ urlpatterns = [
), ),
path( path(
"admin/analytics/", "admin/analytics/",
admin.site.admin_view(analytics), AnalyticsView.as_view(),
name="analytics", name="analytics",
), ),
path("admin/", admin.site.urls), path("admin/", admin.site.urls),

View file

@ -46,7 +46,7 @@
<a href="{% url 'export_data_federal' %}" class="button" role="button"> <a href="{% url 'export_data_federal' %}" class="button" role="button">
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24"> <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use> <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
</svg><span class="margin-left-05">Current Federal</span> </svg><span class="margin-left-05">Current federal</span>
</a> </a>
</li> </li>
</ul> </ul>
@ -87,7 +87,7 @@
<button class="button exportLink" data-export-url="{% url 'export_requests_growth' %}" type="button"> <button class="button exportLink" data-export-url="{% url 'export_requests_growth' %}" type="button">
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24"> <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use> <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
</svg><span class="margin-left-05">Request Growth</span> </svg><span class="margin-left-05">Request growth</span>
</button> </button>
</li> </li>
<li class="usa-button-group__item"> <li class="usa-button-group__item">
@ -132,7 +132,7 @@
data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}" data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}"
data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}" data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}"
> >
<h2>Chart: Unanaged domains</h2> <h2>Chart: Unmanaged domains</h2>
<p>{{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}</p> <p>{{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}</p>
</canvas> </canvas>
</div> </div>

View file

@ -64,12 +64,11 @@
</table> </table>
</div> </div>
{% endfor %} {% endfor %}
{% else %}
<p>{% translate 'You dont have permission to view or edit anything.' %}</p>
{% endif %}
<div class="module module--custom"> <div class="module module--custom">
<h2>Analytics</h2> <h2>Analytics</h2>
<a class="display-block padding-y-1 padding-x-1" href="{% url 'analytics' %}">Dashboard</a> <a class="display-block padding-y-1 padding-x-1" href="{% url 'analytics' %}">Dashboard</a>
</div> </div>
{% else %}
<p>{% translate 'You dont have permission to view or edit anything.' %}</p>
{% endif %}

View file

@ -475,7 +475,7 @@ class AuditedAdminMockData:
class MockDb(TestCase): class MockDb(TestCase):
"""Hardcoded mocks make test case assertions sraightforward.""" """Hardcoded mocks make test case assertions straightforward."""
def setUp(self): def setUp(self):
super().setUp() super().setUp()
@ -622,19 +622,19 @@ class MockDb(TestCase):
) )
with less_console_noise(): with less_console_noise():
self.domain_request_1 = completed_application( self.domain_request_1 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov"
) )
self.domain_request_2 = completed_application( self.domain_request_2 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov"
) )
self.domain_request_3 = completed_application( self.domain_request_3 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov"
) )
self.domain_request_4 = completed_application( self.domain_request_4 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov"
) )
self.domain_request_5 = completed_application( self.domain_request_5 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov"
) )
self.domain_request_3.submit() self.domain_request_3.submit()

View file

@ -5,6 +5,8 @@ from io import StringIO
from registrar.models.domain_request import DomainRequest from registrar.models.domain_request import DomainRequest
from registrar.models.domain import Domain from registrar.models.domain import Domain
from registrar.utility.csv_export import ( from registrar.utility.csv_export import (
export_data_managed_domains_to_csv,
export_data_unmanaged_domains_to_csv,
get_sliced_domains, get_sliced_domains,
get_sliced_requests, get_sliced_requests,
write_domains_csv, write_domains_csv,
@ -530,68 +532,10 @@ class ExportDataTest(MockDb, MockEppLib):
with less_console_noise(): with less_console_noise():
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file) export_data_managed_domains_to_csv(
# Define columns, sort fields, and filter condition csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
columns = [
"Domain name",
"Domain type",
]
sort_fields = [
"domain__name",
]
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": self.start_date,
}
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
# Call the export functions
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(managed_domains_sliced_at_start_date)
writer.writerow([])
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": self.end_date,
}
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date)
writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(managed_domains_sliced_at_end_date)
writer.writerow([])
write_domains_csv(
writer,
columns,
sort_fields,
filter_managed_domains_end_date,
get_domain_managers=True,
should_write_header=True,
) )
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
@ -627,68 +571,10 @@ class ExportDataTest(MockDb, MockEppLib):
with less_console_noise(): with less_console_noise():
# Create a CSV file in memory # Create a CSV file in memory
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file) export_data_unmanaged_domains_to_csv(
# Define columns, sort fields, and filter condition csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
columns = [
"Domain name",
"Domain type",
]
sort_fields = [
"domain__name",
]
filter_unmanaged_domains_start_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": self.start_date,
}
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date)
# Call the export functions
writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(unmanaged_domains_sliced_at_start_date)
writer.writerow([])
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": self.end_date,
}
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date)
writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"])
writer.writerow(
[
"Total",
"Federal",
"Interstate",
"State or territory",
"Tribal",
"County",
"City",
"Special district",
"School district",
"Election office",
]
)
writer.writerow(unmanaged_domains_sliced_at_end_date)
writer.writerow([])
write_domains_csv(
writer,
columns,
sort_fields,
filter_unmanaged_domains_end_date,
get_domain_managers=False,
should_write_header=True,
) )
# Reset the CSV file's position to the beginning # Reset the CSV file's position to the beginning
csv_file.seek(0) csv_file.seek(0)
# Read the content into a variable # Read the content into a variable
@ -696,12 +582,12 @@ class ExportDataTest(MockDb, MockEppLib):
self.maxDiff = None self.maxDiff = None
# We expect the READY domain names with the domain managers: Their counts, and listing at end_date. # We expect the READY domain names with the domain managers: Their counts, and listing at end_date.
expected_content = ( expected_content = (
"UNMANAGED DOMAINS COUNTS AT START DATE\n" "UNMANAGED DOMAINS AT START DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,"
"School district,Election office\n" "School district,Election office\n"
"0,0,0,0,0,0,0,0,0,0\n" "0,0,0,0,0,0,0,0,0,0\n"
"\n" "\n"
"UNMANAGED DOMAINS COUNTS AT END DATE\n" "UNMANAGED DOMAINS AT END DATE\n"
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,"
"School district,Election office\n" "School district,Election office\n"
"1,1,0,0,0,0,0,0,0,0\n" "1,1,0,0,0,0,0,0,0,0\n"
@ -729,16 +615,17 @@ class ExportDataTest(MockDb, MockEppLib):
csv_file = StringIO() csv_file = StringIO()
writer = csv.writer(csv_file) writer = csv.writer(csv_file)
# Define columns, sort fields, and filter condition # Define columns, sort fields, and filter condition
# We'll skip submission date because it's dynamic and therefore
# impossible to set in expected_content
columns = [ columns = [
"Requested domain", "Requested domain",
"Organization type", "Organization type",
"Submission date",
] ]
sort_fields = [ sort_fields = [
"requested_domain__name", "requested_domain__name",
] ]
filter_condition = { filter_condition = {
"status": DomainRequest.RequestStatus.SUBMITTED, "status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": self.end_date, "submission_date__lte": self.end_date,
"submission_date__gte": self.start_date, "submission_date__gte": self.start_date,
} }
@ -750,9 +637,9 @@ class ExportDataTest(MockDb, MockEppLib):
# We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
# and DELETED domains deleted between today-2 and today+2, sorted by deleted then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
expected_content = ( expected_content = (
"Requested domain,Organization type,Submission date\n" "Requested domain,Organization type\n"
"city3.gov,Federal - Executive,2024-03-05\n" "city3.gov,Federal - Executive\n"
"city4.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive\n"
) )
# Normalize line endings and remove commas, # Normalize line endings and remove commas,
@ -785,16 +672,22 @@ class HelperFunctions(MockDb):
"domain__permissions__isnull": False, "domain__permissions__isnull": False,
"domain__first_ready__lte": self.end_date, "domain__first_ready__lte": self.end_date,
} }
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) # Test with distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True)
expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content) self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
# Test without distinct
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition)
expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1]
self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
def test_get_sliced_requests(self): def test_get_sliced_requests(self):
"""Should get fitered requests counts sliced by org type and election office.""" """Should get fitered requests counts sliced by org type and election office."""
with less_console_noise(): with less_console_noise():
filter_condition = { filter_condition = {
"status": DomainRequest.RequestStatus.SUBMITTED, "status": DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": self.end_date, "submission_date__lte": self.end_date,
} }
submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition)

View file

@ -1,3 +1,4 @@
from collections import Counter
import csv import csv
import logging import logging
from datetime import datetime from datetime import datetime
@ -25,7 +26,8 @@ def write_header(writer, columns):
def get_domain_infos(filter_condition, sort_fields): def get_domain_infos(filter_condition, sort_fields):
domain_infos = ( domain_infos = (
DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") DomainInformation.objects.select_related("domain", "authorizing_official")
.prefetch_related("domain__permissions")
.filter(**filter_condition) .filter(**filter_condition)
.order_by(*sort_fields) .order_by(*sort_fields)
.distinct() .distinct()
@ -190,7 +192,7 @@ def write_domains_csv(
def get_requests(filter_condition, sort_fields): def get_requests(filter_condition, sort_fields):
requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
return requests return requests
@ -236,10 +238,10 @@ def write_requests_csv(
"""Receives params from the parent methods and outputs a CSV with filtered and sorted requests. """Receives params from the parent methods and outputs a CSV with filtered and sorted requests.
Works with write_header as long as the same writer object is passed.""" Works with write_header as long as the same writer object is passed."""
all_requetsts = get_requests(filter_condition, sort_fields) all_requests = get_requests(filter_condition, sort_fields)
# Reduce the memory overhead when performing the write operation # Reduce the memory overhead when performing the write operation
paginator = Paginator(all_requetsts, 1000) paginator = Paginator(all_requests, 1000)
for page_num in paginator.page_range: for page_num in paginator.page_range:
page = paginator.page(page_num) page = paginator.page(page_num)
@ -443,26 +445,37 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date):
) )
def get_sliced_domains(filter_condition): def get_sliced_domains(filter_condition, distinct=False):
"""Get fitered domains counts sliced by org type and election office.""" """Get filtered domains counts sliced by org type and election office.
Pass distinct=True when filtering by permissions so we do not to count multiples
when a domain has more that one manager.
"""
domains = DomainInformation.objects.all().filter(**filter_condition).distinct() # Round trip 1: Get distinct domain names based on filter condition
domains_count = domains.count() domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count()
federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count() # Round trip 2: Get counts for other slices
state_or_territory = ( if distinct:
domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() organization_types_query = (
DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct()
) )
tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() else:
county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list(
city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() "organization_type", flat=True
special_district = (
domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
) )
school_district = ( organization_type_counts = Counter(organization_types_query)
domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
) federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)
election_board = domains.filter(is_election_board=True).distinct().count() interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0)
state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0)
tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0)
county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0)
city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0)
special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0)
school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0)
# Round trip 3
election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count()
return [ return [
domains_count, domains_count,
@ -478,26 +491,34 @@ def get_sliced_domains(filter_condition):
] ]
def get_sliced_requests(filter_condition): def get_sliced_requests(filter_condition, distinct=False):
"""Get fitered requests counts sliced by org type and election office.""" """Get filtered requests counts sliced by org type and election office."""
requests = DomainRequest.objects.all().filter(**filter_condition).distinct() # Round trip 1: Get distinct requests based on filter condition
requests_count = requests.count() requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count()
federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() # Round trip 2: Get counts for other slices
state_or_territory = ( if distinct:
requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() organization_types_query = (
DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct()
) )
tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() else:
county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list(
city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() "organization_type", flat=True
special_district = (
requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
) )
school_district = ( organization_type_counts = Counter(organization_types_query)
requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
) federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)
election_board = requests.filter(is_election_board=True).distinct().count() interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0)
state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0)
tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0)
county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0)
city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0)
special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0)
school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0)
# Round trip 3
election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count()
return [ return [
requests_count, requests_count,
@ -531,7 +552,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
"domain__permissions__isnull": False, "domain__permissions__isnull": False,
"domain__first_ready__lte": start_date_formatted, "domain__first_ready__lte": start_date_formatted,
} }
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True)
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
writer.writerow( writer.writerow(
@ -555,7 +576,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
"domain__permissions__isnull": False, "domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted, "domain__first_ready__lte": end_date_formatted,
} }
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True)
writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"])
writer.writerow( writer.writerow(
@ -604,7 +625,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
"domain__permissions__isnull": True, "domain__permissions__isnull": True,
"domain__first_ready__lte": start_date_formatted, "domain__first_ready__lte": start_date_formatted,
} }
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True)
writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) writer.writerow(["UNMANAGED DOMAINS AT START DATE"])
writer.writerow( writer.writerow(
@ -628,7 +649,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
"domain__permissions__isnull": True, "domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted, "domain__first_ready__lte": end_date_formatted,
} }
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True)
writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) writer.writerow(["UNMANAGED DOMAINS AT END DATE"])
writer.writerow( writer.writerow(

View file

@ -2,6 +2,12 @@
from django.http import HttpResponse from django.http import HttpResponse
from django.views import View from django.views import View
from django.shortcuts import render
from django.contrib import admin
from django.db.models import Avg, F
from .. import models
import datetime
from django.utils import timezone
from registrar.utility import csv_export from registrar.utility import csv_export
@ -10,6 +16,129 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class AnalyticsView(View):
def get(self, request):
thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30)
thirty_days_ago = timezone.make_aware(thirty_days_ago)
last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago)
last_30_days_approved_applications = models.DomainRequest.objects.filter(
created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED
)
avg_approval_time = last_30_days_approved_applications.annotate(
approval_time=F("approved_domain__created_at") - F("submission_date")
).aggregate(Avg("approval_time"))["approval_time__avg"]
# Format the timedelta to display only days
if avg_approval_time is not None:
avg_approval_time_display = f"{avg_approval_time.days} days"
else:
avg_approval_time_display = "No approvals to use"
# The start and end dates are passed as url params
start_date = request.GET.get("start_date", "")
end_date = request.GET.get("end_date", "")
start_date_formatted = csv_export.format_start_date(start_date)
end_date_formatted = csv_export.format_end_date(end_date)
filter_managed_domains_start_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": start_date_formatted,
}
filter_managed_domains_end_date = {
"domain__permissions__isnull": False,
"domain__first_ready__lte": end_date_formatted,
}
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True)
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True)
filter_unmanaged_domains_start_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": start_date_formatted,
}
filter_unmanaged_domains_end_date = {
"domain__permissions__isnull": True,
"domain__first_ready__lte": end_date_formatted,
}
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(
filter_unmanaged_domains_start_date, True
)
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True)
filter_ready_domains_start_date = {
"domain__state__in": [models.Domain.State.READY],
"domain__first_ready__lte": start_date_formatted,
}
filter_ready_domains_end_date = {
"domain__state__in": [models.Domain.State.READY],
"domain__first_ready__lte": end_date_formatted,
}
ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date)
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
filter_deleted_domains_start_date = {
"domain__state__in": [models.Domain.State.DELETED],
"domain__deleted__lte": start_date_formatted,
}
filter_deleted_domains_end_date = {
"domain__state__in": [models.Domain.State.DELETED],
"domain__deleted__lte": end_date_formatted,
}
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date)
filter_requests_start_date = {
"created_at__lte": start_date_formatted,
}
filter_requests_end_date = {
"created_at__lte": end_date_formatted,
}
requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date)
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
filter_submitted_requests_start_date = {
"status": models.DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": start_date_formatted,
}
filter_submitted_requests_end_date = {
"status": models.DomainRequest.DomainRequestStatus.SUBMITTED,
"submission_date__lte": end_date_formatted,
}
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date)
context = dict(
# Generate a dictionary of context variables that are common across all admin templates
# (site_header, site_url, ...),
# include it in the larger context dictionary so it's available in the template rendering context.
# This ensures that the admin interface styling and behavior are consistent with other admin pages.
**admin.site.each_context(request),
data=dict(
user_count=models.User.objects.all().count(),
domain_count=models.Domain.objects.all().count(),
ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(),
last_30_days_applications=last_30_days_applications.count(),
last_30_days_approved_applications=last_30_days_approved_applications.count(),
average_application_approval_time_last_30_days=avg_approval_time_display,
managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date,
unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date,
managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date,
unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date,
ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date,
deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date,
ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date,
deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date,
requests_sliced_at_start_date=requests_sliced_at_start_date,
submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date,
requests_sliced_at_end_date=requests_sliced_at_end_date,
submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date,
start_date=start_date,
end_date=end_date,
),
)
return render(request, "admin/analytics.html", context)
class ExportDataType(View): class ExportDataType(View):
def get(self, request, *args, **kwargs): def get(self, request, *args, **kwargs):
# match the CSV example with all the fields # match the CSV example with all the fields