mirror of
https://github.com/cisagov/manage.get.gov.git
synced 2025-05-19 19:09:22 +02:00
Refactors, most notably reduce DB trips in get_sliced methods
This commit is contained in:
Rachid Mrad
parent
8147b5f812
commit
427e110d22
9 changed files with 239 additions and 315 deletions
|
|
@ -1,13 +1,12 @@
|
|||
from datetime import date
|
||||
import logging
|
||||
import datetime
|
||||
import copy
|
||||
|
||||
from django import forms
|
||||
from django.db.models import Avg, F, Value, CharField, Q
|
||||
from django.db.models import Value, CharField, Q
|
||||
from django.db.models.functions import Concat, Coalesce
|
||||
from django.http import HttpResponseRedirect
|
||||
from django.shortcuts import redirect, render
|
||||
from django.shortcuts import redirect
|
||||
from django_fsm import get_available_FIELD_transitions
|
||||
from django.contrib import admin, messages
|
||||
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
||||
|
|
@ -17,7 +16,6 @@ from django.urls import reverse
|
|||
from dateutil.relativedelta import relativedelta # type: ignore
|
||||
from epplibwrapper.errors import ErrorCode, RegistryError
|
||||
from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website
|
||||
from registrar.utility import csv_export
|
||||
from registrar.utility.errors import FSMApplicationError, FSMErrorCodes
|
||||
from registrar.views.utility.mixins import OrderableFieldsMixin
|
||||
from django.contrib.admin.views.main import ORDER_VAR
|
||||
|
|
@ -437,123 +435,6 @@ class UserContactInline(admin.StackedInline):
|
|||
model = models.Contact
|
||||
|
||||
|
||||
def analytics(request):
|
||||
"""View for the reports page."""
|
||||
|
||||
thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30)
|
||||
|
||||
last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago)
|
||||
last_30_days_approved_applications = models.DomainRequest.objects.filter(
|
||||
created_at__gt=thirty_days_ago, status=DomainRequest.DomainRequestStatus.APPROVED
|
||||
)
|
||||
avg_approval_time = last_30_days_approved_applications.annotate(
|
||||
approval_time=F("approved_domain__created_at") - F("submission_date")
|
||||
).aggregate(Avg("approval_time"))["approval_time__avg"]
|
||||
# Format the timedelta to display only days
|
||||
|
||||
if avg_approval_time is not None:
|
||||
avg_approval_time = f"{avg_approval_time.days} days"
|
||||
else:
|
||||
avg_approval_time = "No approvals to use"
|
||||
# The start and end dates are passed as url params
|
||||
start_date = request.GET.get("start_date", "")
|
||||
end_date = request.GET.get("end_date", "")
|
||||
|
||||
start_date_formatted = csv_export.format_start_date(start_date)
|
||||
end_date_formatted = csv_export.format_end_date(end_date)
|
||||
|
||||
filter_managed_domains_start_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_managed_domains_end_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date)
|
||||
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date)
|
||||
|
||||
filter_unmanaged_domains_start_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_unmanaged_domains_end_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date)
|
||||
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date)
|
||||
|
||||
filter_ready_domains_start_date = {
|
||||
"domain__state__in": [Domain.State.READY],
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_ready_domains_end_date = {
|
||||
"domain__state__in": [Domain.State.READY],
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date)
|
||||
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
|
||||
|
||||
filter_deleted_domains_start_date = {
|
||||
"domain__state__in": [Domain.State.DELETED],
|
||||
"domain__deleted__lte": start_date_formatted,
|
||||
}
|
||||
filter_deleted_domains_end_date = {
|
||||
"domain__state__in": [Domain.State.DELETED],
|
||||
"domain__deleted__lte": end_date_formatted,
|
||||
}
|
||||
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
|
||||
deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date)
|
||||
|
||||
filter_requests_start_date = {
|
||||
"created_at__lte": start_date_formatted,
|
||||
}
|
||||
filter_requests_end_date = {
|
||||
"created_at__lte": end_date_formatted,
|
||||
}
|
||||
requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date)
|
||||
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
|
||||
|
||||
filter_submitted_requests_start_date = {
|
||||
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": start_date_formatted,
|
||||
}
|
||||
filter_submitted_requests_end_date = {
|
||||
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": end_date_formatted,
|
||||
}
|
||||
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
|
||||
submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date)
|
||||
|
||||
context = dict(
|
||||
**admin.site.each_context(request),
|
||||
data=dict(
|
||||
user_count=models.User.objects.all().count(),
|
||||
domain_count=models.Domain.objects.all().count(),
|
||||
ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(),
|
||||
last_30_days_applications=last_30_days_applications.count(),
|
||||
last_30_days_approved_applications=last_30_days_approved_applications.count(),
|
||||
average_application_approval_time_last_30_days=avg_approval_time,
|
||||
managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date,
|
||||
unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date,
|
||||
managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date,
|
||||
unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date,
|
||||
ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date,
|
||||
deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date,
|
||||
ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date,
|
||||
deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date,
|
||||
requests_sliced_at_start_date=requests_sliced_at_start_date,
|
||||
submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date,
|
||||
requests_sliced_at_end_date=requests_sliced_at_end_date,
|
||||
submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
),
|
||||
)
|
||||
return render(request, "admin/analytics.html", context)
|
||||
|
||||
|
||||
class MyUserAdmin(BaseUserAdmin):
|
||||
"""Custom user admin class to use our inlines."""
|
||||
|
||||
|
|
|
|||
|
|
@ -330,7 +330,8 @@ CSP_FORM_ACTION = allowed_sources
|
|||
|
||||
# Google analytics requires that we relax our otherwise
|
||||
# strict CSP by allowing scripts to run from their domain
|
||||
# and inline with a nonce, as well as allowing connections back to their domain
|
||||
# and inline with a nonce, as well as allowing connections back to their domain.
|
||||
# Note: If needed, we can embed chart.js instead of using the CDN
|
||||
CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"]
|
||||
CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"]
|
||||
CSP_INCLUDE_NONCE_IN = ["script-src-elem"]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from django.urls import include, path
|
|||
from django.views.generic import RedirectView
|
||||
|
||||
from registrar import views
|
||||
from registrar.admin import analytics
|
||||
from registrar.views.admin_views import (
|
||||
ExportDataDomainsGrowth,
|
||||
ExportDataFederal,
|
||||
|
|
@ -18,6 +17,7 @@ from registrar.views.admin_views import (
|
|||
ExportDataRequestsGrowth,
|
||||
ExportDataType,
|
||||
ExportDataUnmanagedDomains,
|
||||
AnalyticsView,
|
||||
)
|
||||
|
||||
from registrar.views.domain_request import Step
|
||||
|
|
@ -96,7 +96,7 @@ urlpatterns = [
|
|||
),
|
||||
path(
|
||||
"admin/analytics/",
|
||||
admin.site.admin_view(analytics),
|
||||
AnalyticsView.as_view(),
|
||||
name="analytics",
|
||||
),
|
||||
path("admin/", admin.site.urls),
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@
|
|||
<a href="{% url 'export_data_federal' %}" class="button" role="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Current Federal</span>
|
||||
</svg><span class="margin-left-05">Current federal</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -87,7 +87,7 @@
|
|||
<button class="button exportLink" data-export-url="{% url 'export_requests_growth' %}" type="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Request Growth</span>
|
||||
</svg><span class="margin-left-05">Request growth</span>
|
||||
</button>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
|
|
@ -132,7 +132,7 @@
|
|||
data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}"
|
||||
data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}"
|
||||
>
|
||||
<h2>Chart: Unanaged domains</h2>
|
||||
<h2>Chart: Unmanaged domains</h2>
|
||||
<p>{{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}</p>
|
||||
</canvas>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -64,12 +64,11 @@
|
|||
</table>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="module module--custom">
|
||||
<h2>Analytics</h2>
|
||||
<a class="display-block padding-y-1 padding-x-1" href="{% url 'analytics' %}">Dashboard</a>
|
||||
</div>
|
||||
{% else %}
|
||||
<p>{% translate 'You don’t have permission to view or edit anything.' %}</p>
|
||||
{% endif %}
|
||||
|
||||
|
||||
<div class="module module--custom">
|
||||
<h2>Analytics</h2>
|
||||
<a class="display-block padding-y-1 padding-x-1" href="{% url 'analytics' %}">Dashboard</a>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -475,7 +475,7 @@ class AuditedAdminMockData:
|
|||
|
||||
|
||||
class MockDb(TestCase):
|
||||
"""Hardcoded mocks make test case assertions sraightforward."""
|
||||
"""Hardcoded mocks make test case assertions straightforward."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
|
@ -622,19 +622,19 @@ class MockDb(TestCase):
|
|||
)
|
||||
|
||||
with less_console_noise():
|
||||
self.domain_request_1 = completed_application(
|
||||
self.domain_request_1 = completed_domain_request(
|
||||
status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov"
|
||||
)
|
||||
self.domain_request_2 = completed_application(
|
||||
self.domain_request_2 = completed_domain_request(
|
||||
status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov"
|
||||
)
|
||||
self.domain_request_3 = completed_application(
|
||||
self.domain_request_3 = completed_domain_request(
|
||||
status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov"
|
||||
)
|
||||
self.domain_request_4 = completed_application(
|
||||
self.domain_request_4 = completed_domain_request(
|
||||
status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov"
|
||||
)
|
||||
self.domain_request_5 = completed_application(
|
||||
self.domain_request_5 = completed_domain_request(
|
||||
status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov"
|
||||
)
|
||||
self.domain_request_3.submit()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from io import StringIO
|
|||
from registrar.models.domain_request import DomainRequest
|
||||
from registrar.models.domain import Domain
|
||||
from registrar.utility.csv_export import (
|
||||
export_data_managed_domains_to_csv,
|
||||
export_data_unmanaged_domains_to_csv,
|
||||
get_sliced_domains,
|
||||
get_sliced_requests,
|
||||
write_domains_csv,
|
||||
|
|
@ -530,68 +532,10 @@ class ExportDataTest(MockDb, MockEppLib):
|
|||
with less_console_noise():
|
||||
# Create a CSV file in memory
|
||||
csv_file = StringIO()
|
||||
writer = csv.writer(csv_file)
|
||||
# Define columns, sort fields, and filter condition
|
||||
columns = [
|
||||
"Domain name",
|
||||
"Domain type",
|
||||
]
|
||||
sort_fields = [
|
||||
"domain__name",
|
||||
]
|
||||
filter_managed_domains_start_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": self.start_date,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
|
||||
# Call the export functions
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
|
||||
writer.writerow(
|
||||
[
|
||||
"Total",
|
||||
"Federal",
|
||||
"Interstate",
|
||||
"State or territory",
|
||||
"Tribal",
|
||||
"County",
|
||||
"City",
|
||||
"Special district",
|
||||
"School district",
|
||||
"Election office",
|
||||
]
|
||||
)
|
||||
writer.writerow(managed_domains_sliced_at_start_date)
|
||||
writer.writerow([])
|
||||
filter_managed_domains_end_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": self.end_date,
|
||||
}
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date)
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"])
|
||||
writer.writerow(
|
||||
[
|
||||
"Total",
|
||||
"Federal",
|
||||
"Interstate",
|
||||
"State or territory",
|
||||
"Tribal",
|
||||
"County",
|
||||
"City",
|
||||
"Special district",
|
||||
"School district",
|
||||
"Election office",
|
||||
]
|
||||
)
|
||||
writer.writerow(managed_domains_sliced_at_end_date)
|
||||
writer.writerow([])
|
||||
write_domains_csv(
|
||||
writer,
|
||||
columns,
|
||||
sort_fields,
|
||||
filter_managed_domains_end_date,
|
||||
get_domain_managers=True,
|
||||
should_write_header=True,
|
||||
export_data_managed_domains_to_csv(
|
||||
csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
# Reset the CSV file's position to the beginning
|
||||
csv_file.seek(0)
|
||||
# Read the content into a variable
|
||||
|
|
@ -627,68 +571,10 @@ class ExportDataTest(MockDb, MockEppLib):
|
|||
with less_console_noise():
|
||||
# Create a CSV file in memory
|
||||
csv_file = StringIO()
|
||||
writer = csv.writer(csv_file)
|
||||
# Define columns, sort fields, and filter condition
|
||||
columns = [
|
||||
"Domain name",
|
||||
"Domain type",
|
||||
]
|
||||
sort_fields = [
|
||||
"domain__name",
|
||||
]
|
||||
filter_unmanaged_domains_start_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": self.start_date,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date)
|
||||
# Call the export functions
|
||||
writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"])
|
||||
writer.writerow(
|
||||
[
|
||||
"Total",
|
||||
"Federal",
|
||||
"Interstate",
|
||||
"State or territory",
|
||||
"Tribal",
|
||||
"County",
|
||||
"City",
|
||||
"Special district",
|
||||
"School district",
|
||||
"Election office",
|
||||
]
|
||||
)
|
||||
writer.writerow(unmanaged_domains_sliced_at_start_date)
|
||||
writer.writerow([])
|
||||
filter_unmanaged_domains_end_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": self.end_date,
|
||||
}
|
||||
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date)
|
||||
writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"])
|
||||
writer.writerow(
|
||||
[
|
||||
"Total",
|
||||
"Federal",
|
||||
"Interstate",
|
||||
"State or territory",
|
||||
"Tribal",
|
||||
"County",
|
||||
"City",
|
||||
"Special district",
|
||||
"School district",
|
||||
"Election office",
|
||||
]
|
||||
)
|
||||
writer.writerow(unmanaged_domains_sliced_at_end_date)
|
||||
writer.writerow([])
|
||||
write_domains_csv(
|
||||
writer,
|
||||
columns,
|
||||
sort_fields,
|
||||
filter_unmanaged_domains_end_date,
|
||||
get_domain_managers=False,
|
||||
should_write_header=True,
|
||||
export_data_unmanaged_domains_to_csv(
|
||||
csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d")
|
||||
)
|
||||
|
||||
# Reset the CSV file's position to the beginning
|
||||
csv_file.seek(0)
|
||||
# Read the content into a variable
|
||||
|
|
@ -696,12 +582,12 @@ class ExportDataTest(MockDb, MockEppLib):
|
|||
self.maxDiff = None
|
||||
# We expect the READY domain names with the domain managers: Their counts, and listing at end_date.
|
||||
expected_content = (
|
||||
"UNMANAGED DOMAINS COUNTS AT START DATE\n"
|
||||
"UNMANAGED DOMAINS AT START DATE\n"
|
||||
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,"
|
||||
"School district,Election office\n"
|
||||
"0,0,0,0,0,0,0,0,0,0\n"
|
||||
"\n"
|
||||
"UNMANAGED DOMAINS COUNTS AT END DATE\n"
|
||||
"UNMANAGED DOMAINS AT END DATE\n"
|
||||
"Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,"
|
||||
"School district,Election office\n"
|
||||
"1,1,0,0,0,0,0,0,0,0\n"
|
||||
|
|
@ -729,16 +615,17 @@ class ExportDataTest(MockDb, MockEppLib):
|
|||
csv_file = StringIO()
|
||||
writer = csv.writer(csv_file)
|
||||
# Define columns, sort fields, and filter condition
|
||||
# We'll skip submission date because it's dynamic and therefore
|
||||
# impossible to set in expected_content
|
||||
columns = [
|
||||
"Requested domain",
|
||||
"Organization type",
|
||||
"Submission date",
|
||||
]
|
||||
sort_fields = [
|
||||
"requested_domain__name",
|
||||
]
|
||||
filter_condition = {
|
||||
"status": DomainRequest.RequestStatus.SUBMITTED,
|
||||
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": self.end_date,
|
||||
"submission_date__gte": self.start_date,
|
||||
}
|
||||
|
|
@ -750,9 +637,9 @@ class ExportDataTest(MockDb, MockEppLib):
|
|||
# We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
|
||||
# and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
|
||||
expected_content = (
|
||||
"Requested domain,Organization type,Submission date\n"
|
||||
"city3.gov,Federal - Executive,2024-03-05\n"
|
||||
"city4.gov,Federal - Executive,2024-03-05\n"
|
||||
"Requested domain,Organization type\n"
|
||||
"city3.gov,Federal - Executive\n"
|
||||
"city4.gov,Federal - Executive\n"
|
||||
)
|
||||
|
||||
# Normalize line endings and remove commas,
|
||||
|
|
@ -785,16 +672,22 @@ class HelperFunctions(MockDb):
|
|||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": self.end_date,
|
||||
}
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition)
|
||||
# Test with distinct
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True)
|
||||
expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1]
|
||||
self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
|
||||
|
||||
# Test without distinct
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition)
|
||||
expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1]
|
||||
self.assertEqual(managed_domains_sliced_at_end_date, expected_content)
|
||||
|
||||
def test_get_sliced_requests(self):
|
||||
"""Should get fitered requests counts sliced by org type and election office."""
|
||||
|
||||
with less_console_noise():
|
||||
filter_condition = {
|
||||
"status": DomainRequest.RequestStatus.SUBMITTED,
|
||||
"status": DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": self.end_date,
|
||||
}
|
||||
submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections import Counter
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
|
@ -25,7 +26,8 @@ def write_header(writer, columns):
|
|||
|
||||
def get_domain_infos(filter_condition, sort_fields):
|
||||
domain_infos = (
|
||||
DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions")
|
||||
DomainInformation.objects.select_related("domain", "authorizing_official")
|
||||
.prefetch_related("domain__permissions")
|
||||
.filter(**filter_condition)
|
||||
.order_by(*sort_fields)
|
||||
.distinct()
|
||||
|
|
@ -190,7 +192,7 @@ def write_domains_csv(
|
|||
|
||||
|
||||
def get_requests(filter_condition, sort_fields):
|
||||
requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct()
|
||||
requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
|
||||
return requests
|
||||
|
||||
|
||||
|
|
@ -236,10 +238,10 @@ def write_requests_csv(
|
|||
"""Receives params from the parent methods and outputs a CSV with filtered and sorted requests.
|
||||
Works with write_header as long as the same writer object is passed."""
|
||||
|
||||
all_requetsts = get_requests(filter_condition, sort_fields)
|
||||
all_requests = get_requests(filter_condition, sort_fields)
|
||||
|
||||
# Reduce the memory overhead when performing the write operation
|
||||
paginator = Paginator(all_requetsts, 1000)
|
||||
paginator = Paginator(all_requests, 1000)
|
||||
|
||||
for page_num in paginator.page_range:
|
||||
page = paginator.page(page_num)
|
||||
|
|
@ -443,26 +445,37 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date):
|
|||
)
|
||||
|
||||
|
||||
def get_sliced_domains(filter_condition):
|
||||
"""Get fitered domains counts sliced by org type and election office."""
|
||||
def get_sliced_domains(filter_condition, distinct=False):
|
||||
"""Get filtered domains counts sliced by org type and election office.
|
||||
Pass distinct=True when filtering by permissions so we do not to count multiples
|
||||
when a domain has more that one manager.
|
||||
"""
|
||||
|
||||
domains = DomainInformation.objects.all().filter(**filter_condition).distinct()
|
||||
domains_count = domains.count()
|
||||
federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
|
||||
interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count()
|
||||
state_or_territory = (
|
||||
domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
|
||||
)
|
||||
tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
|
||||
county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
|
||||
city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
|
||||
special_district = (
|
||||
domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
|
||||
)
|
||||
school_district = (
|
||||
domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
|
||||
)
|
||||
election_board = domains.filter(is_election_board=True).distinct().count()
|
||||
# Round trip 1: Get distinct domain names based on filter condition
|
||||
domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count()
|
||||
|
||||
# Round trip 2: Get counts for other slices
|
||||
if distinct:
|
||||
organization_types_query = (
|
||||
DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct()
|
||||
)
|
||||
else:
|
||||
organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list(
|
||||
"organization_type", flat=True
|
||||
)
|
||||
organization_type_counts = Counter(organization_types_query)
|
||||
|
||||
federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)
|
||||
interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0)
|
||||
state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0)
|
||||
tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0)
|
||||
county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0)
|
||||
city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0)
|
||||
special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0)
|
||||
school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0)
|
||||
|
||||
# Round trip 3
|
||||
election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count()
|
||||
|
||||
return [
|
||||
domains_count,
|
||||
|
|
@ -478,26 +491,34 @@ def get_sliced_domains(filter_condition):
|
|||
]
|
||||
|
||||
|
||||
def get_sliced_requests(filter_condition):
|
||||
"""Get fitered requests counts sliced by org type and election office."""
|
||||
def get_sliced_requests(filter_condition, distinct=False):
|
||||
"""Get filtered requests counts sliced by org type and election office."""
|
||||
|
||||
requests = DomainRequest.objects.all().filter(**filter_condition).distinct()
|
||||
requests_count = requests.count()
|
||||
federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count()
|
||||
interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count()
|
||||
state_or_territory = (
|
||||
requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count()
|
||||
)
|
||||
tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count()
|
||||
county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count()
|
||||
city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count()
|
||||
special_district = (
|
||||
requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count()
|
||||
)
|
||||
school_district = (
|
||||
requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count()
|
||||
)
|
||||
election_board = requests.filter(is_election_board=True).distinct().count()
|
||||
# Round trip 1: Get distinct requests based on filter condition
|
||||
requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count()
|
||||
|
||||
# Round trip 2: Get counts for other slices
|
||||
if distinct:
|
||||
organization_types_query = (
|
||||
DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct()
|
||||
)
|
||||
else:
|
||||
organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list(
|
||||
"organization_type", flat=True
|
||||
)
|
||||
organization_type_counts = Counter(organization_types_query)
|
||||
|
||||
federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0)
|
||||
interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0)
|
||||
state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0)
|
||||
tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0)
|
||||
county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0)
|
||||
city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0)
|
||||
special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0)
|
||||
school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0)
|
||||
|
||||
# Round trip 3
|
||||
election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count()
|
||||
|
||||
return [
|
||||
requests_count,
|
||||
|
|
@ -531,7 +552,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
|
|||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
|
||||
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True)
|
||||
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"])
|
||||
writer.writerow(
|
||||
|
|
@ -555,7 +576,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date):
|
|||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date)
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True)
|
||||
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"])
|
||||
writer.writerow(
|
||||
|
|
@ -604,7 +625,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
|
|||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date)
|
||||
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True)
|
||||
|
||||
writer.writerow(["UNMANAGED DOMAINS AT START DATE"])
|
||||
writer.writerow(
|
||||
|
|
@ -628,7 +649,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date):
|
|||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date)
|
||||
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True)
|
||||
|
||||
writer.writerow(["UNMANAGED DOMAINS AT END DATE"])
|
||||
writer.writerow(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,12 @@
|
|||
|
||||
from django.http import HttpResponse
|
||||
from django.views import View
|
||||
from django.shortcuts import render
|
||||
from django.contrib import admin
|
||||
from django.db.models import Avg, F
|
||||
from .. import models
|
||||
import datetime
|
||||
from django.utils import timezone
|
||||
|
||||
from registrar.utility import csv_export
|
||||
|
||||
|
|
@ -10,6 +16,129 @@ import logging
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnalyticsView(View):
|
||||
def get(self, request):
|
||||
thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30)
|
||||
thirty_days_ago = timezone.make_aware(thirty_days_ago)
|
||||
|
||||
last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago)
|
||||
last_30_days_approved_applications = models.DomainRequest.objects.filter(
|
||||
created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED
|
||||
)
|
||||
avg_approval_time = last_30_days_approved_applications.annotate(
|
||||
approval_time=F("approved_domain__created_at") - F("submission_date")
|
||||
).aggregate(Avg("approval_time"))["approval_time__avg"]
|
||||
# Format the timedelta to display only days
|
||||
if avg_approval_time is not None:
|
||||
avg_approval_time_display = f"{avg_approval_time.days} days"
|
||||
else:
|
||||
avg_approval_time_display = "No approvals to use"
|
||||
|
||||
# The start and end dates are passed as url params
|
||||
start_date = request.GET.get("start_date", "")
|
||||
end_date = request.GET.get("end_date", "")
|
||||
|
||||
start_date_formatted = csv_export.format_start_date(start_date)
|
||||
end_date_formatted = csv_export.format_end_date(end_date)
|
||||
|
||||
filter_managed_domains_start_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_managed_domains_end_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True)
|
||||
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True)
|
||||
|
||||
filter_unmanaged_domains_start_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_unmanaged_domains_end_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(
|
||||
filter_unmanaged_domains_start_date, True
|
||||
)
|
||||
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True)
|
||||
|
||||
filter_ready_domains_start_date = {
|
||||
"domain__state__in": [models.Domain.State.READY],
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
filter_ready_domains_end_date = {
|
||||
"domain__state__in": [models.Domain.State.READY],
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date)
|
||||
ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date)
|
||||
|
||||
filter_deleted_domains_start_date = {
|
||||
"domain__state__in": [models.Domain.State.DELETED],
|
||||
"domain__deleted__lte": start_date_formatted,
|
||||
}
|
||||
filter_deleted_domains_end_date = {
|
||||
"domain__state__in": [models.Domain.State.DELETED],
|
||||
"domain__deleted__lte": end_date_formatted,
|
||||
}
|
||||
deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date)
|
||||
deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date)
|
||||
|
||||
filter_requests_start_date = {
|
||||
"created_at__lte": start_date_formatted,
|
||||
}
|
||||
filter_requests_end_date = {
|
||||
"created_at__lte": end_date_formatted,
|
||||
}
|
||||
requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date)
|
||||
requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date)
|
||||
|
||||
filter_submitted_requests_start_date = {
|
||||
"status": models.DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": start_date_formatted,
|
||||
}
|
||||
filter_submitted_requests_end_date = {
|
||||
"status": models.DomainRequest.DomainRequestStatus.SUBMITTED,
|
||||
"submission_date__lte": end_date_formatted,
|
||||
}
|
||||
submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date)
|
||||
submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date)
|
||||
|
||||
context = dict(
|
||||
# Generate a dictionary of context variables that are common across all admin templates
|
||||
# (site_header, site_url, ...),
|
||||
# include it in the larger context dictionary so it's available in the template rendering context.
|
||||
# This ensures that the admin interface styling and behavior are consistent with other admin pages.
|
||||
**admin.site.each_context(request),
|
||||
data=dict(
|
||||
user_count=models.User.objects.all().count(),
|
||||
domain_count=models.Domain.objects.all().count(),
|
||||
ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(),
|
||||
last_30_days_applications=last_30_days_applications.count(),
|
||||
last_30_days_approved_applications=last_30_days_approved_applications.count(),
|
||||
average_application_approval_time_last_30_days=avg_approval_time_display,
|
||||
managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date,
|
||||
unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date,
|
||||
managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date,
|
||||
unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date,
|
||||
ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date,
|
||||
deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date,
|
||||
ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date,
|
||||
deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date,
|
||||
requests_sliced_at_start_date=requests_sliced_at_start_date,
|
||||
submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date,
|
||||
requests_sliced_at_end_date=requests_sliced_at_end_date,
|
||||
submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
),
|
||||
)
|
||||
return render(request, "admin/analytics.html", context)
|
||||
|
||||
|
||||
class ExportDataType(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# match the CSV example with all the fields
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue