mirror of
https://github.com/cisagov/manage.get.gov.git
synced 2025-05-29 17:00:02 +02:00
Reports, chart wip
This commit is contained in:
Rachid Mrad
parent
f8a95564a9
commit
0a174e5d29
12 changed files with 498 additions and 140 deletions
|
|
@ -27,6 +27,7 @@ from django_fsm import TransitionNotAllowed # type: ignore
|
|||
from django.utils.safestring import mark_safe
|
||||
from django.utils.html import escape
|
||||
from django.contrib.auth.forms import UserChangeForm, UsernameField
|
||||
from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType
|
||||
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
|
@ -363,6 +364,75 @@ class UserContactInline(admin.StackedInline):
|
|||
model = models.Contact
|
||||
|
||||
|
||||
def user_analytics(request):
|
||||
|
||||
end_date = datetime.datetime.today()
|
||||
start_date = datetime.datetime.today() - datetime.timedelta(days=30)
|
||||
|
||||
last_30_days_applications = models.DomainApplication.objects.filter(
|
||||
created_at__gt=start_date
|
||||
)
|
||||
avg_approval_time = last_30_days_applications.annotate(
|
||||
approval_time=F("approved_domain__created_at") - F("created_at")
|
||||
).aggregate(Avg("approval_time"))["approval_time__avg"]
|
||||
# format the timedelta?
|
||||
avg_approval_time = str(avg_approval_time)
|
||||
|
||||
start_date = request.GET.get("start_date", "")
|
||||
end_date = request.GET.get("end_date", "")
|
||||
|
||||
start_date_formatted = csv_export.format_start_date(start_date)
|
||||
end_date_formatted = csv_export.format_end_date(end_date)
|
||||
|
||||
filter_managed_domains_start_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__created_at__lte": start_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date)
|
||||
managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5]
|
||||
|
||||
logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}")
|
||||
|
||||
filter_unmanaged_domains_start_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date)
|
||||
unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5]
|
||||
filter_managed_domains_end_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date)
|
||||
managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4]
|
||||
filter_unmanaged_domains_end_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date)
|
||||
unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5]
|
||||
|
||||
# get number of ready domains, counts by org type and election office
|
||||
# add to context
|
||||
|
||||
# get number of submitted request counts by org type and election office
|
||||
# add to context
|
||||
|
||||
context = dict(
|
||||
**admin.site.each_context(request),
|
||||
data=dict(
|
||||
user_count=models.User.objects.all().count(),
|
||||
domain_count=models.Domain.objects.all().count(),
|
||||
applications_last_30_days=last_30_days_applications.count(),
|
||||
average_application_approval_time_last_30_days=avg_approval_time,
|
||||
managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date,
|
||||
unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date,
|
||||
managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date,
|
||||
unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date,
|
||||
),
|
||||
)
|
||||
return render(request, "admin/analytics.html", context)
|
||||
|
||||
class MyUserAdmin(BaseUserAdmin):
|
||||
"""Custom user admin class to use our inlines."""
|
||||
|
||||
|
|
@ -464,79 +534,6 @@ class MyUserAdmin(BaseUserAdmin):
|
|||
# in autocomplete_fields for user
|
||||
ordering = ["first_name", "last_name", "email"]
|
||||
|
||||
def get_urls(self):
|
||||
"""Map a new page in admin for analytics."""
|
||||
urlpatterns = super().get_urls()
|
||||
|
||||
# Used to extrapolate a path name, for instance
|
||||
# name="{app_label}_{model_name}_export_data_type"
|
||||
domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name
|
||||
|
||||
my_urls = [
|
||||
path(
|
||||
"analytics/",
|
||||
self.admin_site.admin_view(self.user_analytics),
|
||||
name="user_analytics",
|
||||
),
|
||||
path(
|
||||
"export_data_type/",
|
||||
self.export_data_type,
|
||||
name="%s_%s_export_data_type" % domain_path_meta,
|
||||
),
|
||||
path(
|
||||
"export_data_full/",
|
||||
self.export_data_full,
|
||||
name="%s_%s_export_data_full" % domain_path_meta,
|
||||
),
|
||||
path(
|
||||
"export_data_federal/",
|
||||
self.export_data_federal,
|
||||
name="%s_%s_export_data_federal" % domain_path_meta,
|
||||
),
|
||||
]
|
||||
|
||||
return my_urls + urlpatterns
|
||||
|
||||
def export_data_type(self, request):
|
||||
# match the CSV example with all the fields
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"'
|
||||
csv_export.export_data_type_to_csv(response)
|
||||
return response
|
||||
|
||||
def export_data_full(self, request):
|
||||
# Smaller export based on 1
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="current-full.csv"'
|
||||
csv_export.export_data_full_to_csv(response)
|
||||
return response
|
||||
|
||||
def export_data_federal(self, request):
|
||||
# Federal only
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="current-federal.csv"'
|
||||
csv_export.export_data_federal_to_csv(response)
|
||||
return response
|
||||
|
||||
def user_analytics(self, request):
|
||||
last_30_days_applications = models.DomainApplication.objects.filter(
|
||||
created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30)
|
||||
)
|
||||
avg_approval_time = last_30_days_applications.annotate(
|
||||
approval_time=F("approved_domain__created_at") - F("created_at")
|
||||
).aggregate(Avg("approval_time"))["approval_time__avg"]
|
||||
# format the timedelta?
|
||||
avg_approval_time = str(avg_approval_time)
|
||||
context = dict(
|
||||
**self.admin_site.each_context(request),
|
||||
data=dict(
|
||||
user_count=models.User.objects.all().count(),
|
||||
domain_count=models.Domain.objects.all().count(),
|
||||
applications_last_30_days=last_30_days_applications.count(),
|
||||
average_application_approval_time_last_30_days=avg_approval_time,
|
||||
),
|
||||
)
|
||||
return render(request, "admin/analytics.html", context)
|
||||
def get_search_results(self, request, queryset, search_term):
|
||||
"""
|
||||
Override for get_search_results. This affects any upstream model using autocomplete_fields,
|
||||
|
|
|
|||
|
|
@ -322,17 +322,18 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk,
|
|||
// Default the value of the end date input field to the current date
|
||||
let endDateInput =document.getElementById('end');
|
||||
|
||||
let exportGrowthReportButton = document.getElementById('exportLink');
|
||||
let exportButtons = document.querySelectorAll('.exportLink');
|
||||
|
||||
if (exportGrowthReportButton) {
|
||||
if (exportButtons.length > 0) {
|
||||
startDateInput.value = currentDate;
|
||||
endDateInput.value = currentDate;
|
||||
|
||||
exportGrowthReportButton.addEventListener('click', function() {
|
||||
exportButtons.forEach((btn) => {
|
||||
btn.addEventListener('click', function() {
|
||||
// Get the selected start and end dates
|
||||
let startDate = startDateInput.value;
|
||||
let endDate = endDateInput.value;
|
||||
let exportUrl = document.getElementById('exportLink').dataset.exportUrl;
|
||||
let exportUrl = btn.dataset.exportUrl;
|
||||
|
||||
// Build the URL with parameters
|
||||
exportUrl += "?start_date=" + startDate + "&end_date=" + endDate;
|
||||
|
|
@ -340,6 +341,7 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk,
|
|||
// Redirect to the export URL
|
||||
window.location.href = exportUrl;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -303,3 +303,20 @@ input.admin-confirm-button {
|
|||
display: contents !important;
|
||||
}
|
||||
}
|
||||
|
||||
.usa-button-group {
|
||||
margin-left: -0.25rem!important;
|
||||
padding-left: 0!important;
|
||||
.usa-button-group__item {
|
||||
list-style-type: none;
|
||||
line-height: normal;
|
||||
}
|
||||
.button {
|
||||
display: inline-block;
|
||||
padding: 10px 8px;
|
||||
line-height: normal;
|
||||
}
|
||||
.usa-icon {
|
||||
top: 2px;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -330,7 +330,7 @@ CSP_FORM_ACTION = allowed_sources
|
|||
# Google analytics requires that we relax our otherwise
|
||||
# strict CSP by allowing scripts to run from their domain
|
||||
# and inline with a nonce, as well as allowing connections back to their domain
|
||||
CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"]
|
||||
CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"]
|
||||
CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"]
|
||||
CSP_INCLUDE_NONCE_IN = ["script-src-elem"]
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,8 @@ from django.urls import include, path
|
|||
from django.views.generic import RedirectView
|
||||
|
||||
from registrar import views
|
||||
|
||||
from registrar.views.admin_views import ExportData
|
||||
|
||||
from registrar.admin import user_analytics
|
||||
from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType
|
||||
|
||||
from registrar.views.application import Step
|
||||
from registrar.views.utility import always_404
|
||||
|
|
@ -52,7 +51,42 @@ urlpatterns = [
|
|||
"admin/logout/",
|
||||
RedirectView.as_view(pattern_name="logout", permanent=False),
|
||||
),
|
||||
path("export_data/", ExportData.as_view(), name="admin_export_data"),
|
||||
path(
|
||||
"admin/analytics/export_data_type/",
|
||||
ExportDataType.as_view(),
|
||||
name="export_data_type",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/export_data_full/",
|
||||
ExportDataFull.as_view(),
|
||||
name="export_data_full",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/export_data_federal/",
|
||||
ExportDataFederal.as_view(),
|
||||
name="export_data_federal",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/export_domain_growth/",
|
||||
ExportDataDomainGrowth.as_view(),
|
||||
name="export_domain_growth",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/export_managed_unmanaged/",
|
||||
ExportDataManagedVsUnmanaged.as_view(),
|
||||
name="export_managed_unmanaged",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/export_requests/",
|
||||
ExportDataRequests.as_view(),
|
||||
name="export_requests",
|
||||
),
|
||||
path(
|
||||
"admin/analytics/",
|
||||
admin.site.admin_view(user_analytics),
|
||||
name="user_analytics",
|
||||
),
|
||||
|
||||
path("admin/", admin.site.urls),
|
||||
path(
|
||||
"application/<id>/edit/",
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ def handle_profile(sender, instance, **kwargs):
|
|||
last_name = getattr(instance, "last_name", "")
|
||||
email = getattr(instance, "email", "")
|
||||
phone = getattr(instance, "phone", "")
|
||||
logger.info(f'in handle_profile first {instance}')
|
||||
|
||||
is_new_user = kwargs.get("created", False)
|
||||
|
||||
|
|
@ -36,6 +37,7 @@ def handle_profile(sender, instance, **kwargs):
|
|||
contacts = Contact.objects.filter(user=instance)
|
||||
|
||||
if len(contacts) == 0: # no matching contact
|
||||
logger.info(f'inside no matching contacts for first {first_name} last {last_name} email {email}')
|
||||
Contact.objects.create(
|
||||
user=instance,
|
||||
first_name=first_name,
|
||||
|
|
|
|||
|
|
@ -1,25 +1,11 @@
|
|||
{% extends "admin/base_site.html" %}
|
||||
|
||||
{% load static %}
|
||||
|
||||
|
||||
{% block content_title %}<h1>Registrar Analytics</h1>{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
||||
{% block object-tools %}
|
||||
<ul class="object-tools">
|
||||
<li>
|
||||
<a href="{% url 'admin:registrar_domain_export_data_type' %}" class="button">Export all domain metadata</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{% url 'admin:registrar_domain_export_data_full' %}" class="button">Export current-full.csv</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{% url 'admin:registrar_domain_export_data_federal' %}" class="button">Export current-federal.csv</a>
|
||||
</li>
|
||||
</ul>
|
||||
{% endblock %}
|
||||
|
||||
<div id="content-main" class="analytics">
|
||||
<div class="module">
|
||||
<h2>At a glance</h2>
|
||||
|
|
@ -34,17 +20,46 @@
|
|||
</div>
|
||||
|
||||
<div class="module">
|
||||
<h2>Domain growth</h2>
|
||||
<h2>Current domains</h2>
|
||||
<div class="padding-2">
|
||||
<ul class="usa-button-group">
|
||||
<li class="usa-button-group__item">
|
||||
<a href="{% url 'export_data_type' %}" class="button" role="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">All domain metadata</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
<a href="{% url 'export_data_full' %}" class="button" role="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Current full</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
<a href="{% url 'export_data_federal' %}" class="button" role="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Current Federal</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="module">
|
||||
<h2>Growth reports</h2>
|
||||
<div class="padding-2">
|
||||
{% comment %}
|
||||
Inputs of type date suck for accessibility.
|
||||
We'll need to replace those guys with a django form once we figure out how to hook one onto this page.
|
||||
The challenge is in the path definition in urls. It does NOT like admin/export_data/
|
||||
The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/
|
||||
|
||||
See the commit "Review for ticket #999"
|
||||
{% endcomment %}
|
||||
<div class="display-flex flex-align-baseline flex-justify margin-y-1">
|
||||
<div>
|
||||
<div class="display-flex flex-align-baseline margin-top-1 margin-bottom-2">
|
||||
<div class="margin-right-1">
|
||||
<label for="start">Start date:</label>
|
||||
<input type="date" id="start" name="start" value="2018-07-22" min="2018-01-01" />
|
||||
</div>
|
||||
|
|
@ -52,8 +67,58 @@
|
|||
<label for="end">End date:</label>
|
||||
<input type="date" id="end" name="end" value="2023-12-01" min="2023-12-01" />
|
||||
</div>
|
||||
<button id="exportLink" data-export-url="{% url 'admin_export_data' %}" type="button" class="button">Export</button>
|
||||
</div>
|
||||
|
||||
<ul class="usa-button-group">
|
||||
<li class="usa-button-group__item">
|
||||
<button class="button exportLink" data-export-url="{% url 'export_domain_growth' %}" type="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Domain growth</span>
|
||||
</button>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
<button class="button exportLink" data-export-url="{% url 'export_requests' %}" type="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Request Growth</span>
|
||||
</button>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
<button class="button exportLink" data-export-url="{% url 'export_managed_unmanaged' %}" type="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
|
||||
</svg><span class="margin-left-05">Managed vs unmanaged domain</span>
|
||||
</button>
|
||||
</li>
|
||||
<li class="usa-button-group__item">
|
||||
<button class="button exportLink" data-export-url="{% url 'user_analytics' %}" type="button">
|
||||
<svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
|
||||
<use xlink:href="{%static 'img/sprite.svg'%}#assessment"></use>
|
||||
</svg><span class="margin-left-05">Update charts</span>
|
||||
</button>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<div class="grid-row grid-gap-2 margin-y-2">
|
||||
<div class="grid-col">
|
||||
<canvas id="myChart" width="400" height="200"
|
||||
data-list-one="{{data.managed_domains_sliced_at_start_date}}"
|
||||
data-list-two="{{data.unmanaged_domains_sliced_at_start_date}}"
|
||||
data-list-three="{{data.managed_domains_sliced_at_end_date}}"
|
||||
data-list-four="{{data.unmanaged_domains_sliced_at_end_date}}"
|
||||
></canvas>
|
||||
</div>
|
||||
<div class="grid-col">
|
||||
<canvas id="myChart2" width="400" height="200"
|
||||
data-list-one="{{data.managed_domains_sliced_at_start_date}}"
|
||||
data-list-two="{{data.unmanaged_domains_sliced_at_start_date}}"
|
||||
data-list-three="{{data.managed_domains_sliced_at_end_date}}"
|
||||
data-list-four="{{data.unmanaged_domains_sliced_at_end_date}}"
|
||||
></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@
|
|||
>
|
||||
<script src="{% static 'js/uswds-init.min.js' %}" defer></script>
|
||||
<script src="{% static 'js/uswds.min.js' %}" defer></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<script type="application/javascript" src="{% static 'js/get-gov-admin.js' %}" defer></script>
|
||||
<script type="application/javascript" src="{% static 'js/get-gov-reports.js' %}" defer></script>
|
||||
{% endblock %}
|
||||
|
||||
{% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class TestViews(TestCase):
|
|||
|
||||
# Construct the URL for the export data view with start_date and end_date parameters:
|
||||
# This stuff is currently done in JS
|
||||
export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}"
|
||||
export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}"
|
||||
|
||||
# Make a GET request to the export data page
|
||||
response = self.client.get(export_data_url)
|
||||
|
|
|
|||
|
|
@ -542,7 +542,7 @@ class ExportDataTest(MockEppLib):
|
|||
are pulled when the growth report conditions are applied to export_domains_to_writed.
|
||||
Test that ready domains are sorted by first_ready/deleted dates first, names second.
|
||||
|
||||
We considered testing export_data_growth_to_csv which calls write_body
|
||||
We considered testing export_data_domain_growth_to_csv which calls write_body
|
||||
and would have been easy to set up, but expected_content would contain created_at dates
|
||||
which are hard to mock.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import csv
|
|||
import logging
|
||||
from datetime import datetime
|
||||
from registrar.models.domain import Domain
|
||||
from registrar.models.domain_application import DomainApplication
|
||||
from registrar.models.domain_information import DomainInformation
|
||||
from django.utils import timezone
|
||||
from django.core.paginator import Paginator
|
||||
|
|
@ -19,10 +20,8 @@ def write_header(writer, columns):
|
|||
Receives params from the parent methods and outputs a CSV with a header row.
|
||||
Works with write_header as long as the same writer object is passed.
|
||||
"""
|
||||
|
||||
writer.writerow(columns)
|
||||
|
||||
|
||||
def get_domain_infos(filter_condition, sort_fields):
|
||||
domain_infos = (
|
||||
DomainInformation.objects.select_related("domain", "authorizing_official")
|
||||
|
|
@ -43,7 +42,6 @@ def get_domain_infos(filter_condition, sort_fields):
|
|||
)
|
||||
return domain_infos_cleaned
|
||||
|
||||
|
||||
def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False):
|
||||
"""Given a set of columns, generate a new row from cleaned column data"""
|
||||
|
||||
|
|
@ -104,7 +102,6 @@ def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None
|
|||
row = [FIELDS.get(column, "") for column in columns]
|
||||
return row
|
||||
|
||||
|
||||
def _get_security_emails(sec_contact_ids):
|
||||
"""
|
||||
Retrieve security contact emails for the given security contact IDs.
|
||||
|
|
@ -126,7 +123,6 @@ def _get_security_emails(sec_contact_ids):
|
|||
|
||||
return security_emails_dict
|
||||
|
||||
|
||||
def update_columns_with_domain_managers(columns, max_dm_count):
|
||||
"""
|
||||
Update the columns list to include "Domain manager email {#}" headers
|
||||
|
|
@ -135,7 +131,6 @@ def update_columns_with_domain_managers(columns, max_dm_count):
|
|||
for i in range(1, max_dm_count + 1):
|
||||
columns.append(f"Domain manager email {i}")
|
||||
|
||||
|
||||
def write_csv(
|
||||
writer,
|
||||
columns,
|
||||
|
|
@ -148,7 +143,7 @@ def write_csv(
|
|||
Receives params from the parent methods and outputs a CSV with fltered and sorted domains.
|
||||
Works with write_header as longas the same writer object is passed.
|
||||
get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv
|
||||
should_write_header: Conditional bc export_data_growth_to_csv calls write_body twice
|
||||
should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice
|
||||
"""
|
||||
|
||||
all_domain_infos = get_domain_infos(filter_condition, sort_fields)
|
||||
|
|
@ -158,15 +153,15 @@ def write_csv(
|
|||
|
||||
security_emails_dict = _get_security_emails(sec_contact_ids)
|
||||
|
||||
# Reduce the memory overhead when performing the write operation
|
||||
paginator = Paginator(all_domain_infos, 1000)
|
||||
|
||||
if get_domain_managers and len(all_domain_infos) > 0:
|
||||
# We want to get the max amont of domain managers an
|
||||
# account has to set the column header dynamically
|
||||
max_dm_count = max(len(domain_info.domain.permissions.all()) for domain_info in all_domain_infos)
|
||||
update_columns_with_domain_managers(columns, max_dm_count)
|
||||
|
||||
# Reduce the memory overhead when performing the write operation
|
||||
paginator = Paginator(all_domain_infos, 1000)
|
||||
|
||||
for page_num in paginator.page_range:
|
||||
page = paginator.page(page_num)
|
||||
rows = []
|
||||
|
|
@ -185,6 +180,82 @@ def write_csv(
|
|||
|
||||
writer.writerows(rows)
|
||||
|
||||
def get_domain_requests(filter_condition, sort_fields):
|
||||
domain_requests = (
|
||||
DomainApplication.objects.all()
|
||||
.filter(**filter_condition)
|
||||
.order_by(*sort_fields)
|
||||
)
|
||||
|
||||
return domain_requests
|
||||
|
||||
def parse_request_row(columns, request: DomainApplication):
|
||||
"""Given a set of columns, generate a new row from cleaned column data"""
|
||||
|
||||
requested_domain_name = 'No requested domain'
|
||||
|
||||
# Domain should never be none when parsing this information
|
||||
if request.requested_domain is not None:
|
||||
domain = request.requested_domain
|
||||
requested_domain_name = domain.name
|
||||
|
||||
domain = request.requested_domain # type: ignore
|
||||
|
||||
if request.federal_type:
|
||||
request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}"
|
||||
else:
|
||||
request_type = request.get_organization_type_display()
|
||||
|
||||
# create a dictionary of fields which can be included in output
|
||||
FIELDS = {
|
||||
"Requested domain": requested_domain_name,
|
||||
"Status": request.get_status_display(),
|
||||
"Organization type": request_type,
|
||||
"Agency": request.federal_agency,
|
||||
"Organization name": request.organization_name,
|
||||
"City": request.city,
|
||||
"State": request.state_territory,
|
||||
"AO email": request.authorizing_official.email if request.authorizing_official else " ",
|
||||
"Security contact email": request,
|
||||
"Created at": request.created_at,
|
||||
"Submission date": request.submission_date,
|
||||
}
|
||||
|
||||
row = [FIELDS.get(column, "") for column in columns]
|
||||
return row
|
||||
|
||||
def write_requests_csv(
|
||||
writer,
|
||||
columns,
|
||||
sort_fields,
|
||||
filter_condition,
|
||||
should_write_header=True,
|
||||
):
|
||||
"""
|
||||
"""
|
||||
|
||||
all_requetsts = get_domain_requests(filter_condition, sort_fields)
|
||||
|
||||
# Reduce the memory overhead when performing the write operation
|
||||
paginator = Paginator(all_requetsts, 1000)
|
||||
|
||||
for page_num in paginator.page_range:
|
||||
page = paginator.page(page_num)
|
||||
rows = []
|
||||
for request in page.object_list:
|
||||
try:
|
||||
row = parse_request_row(columns, request)
|
||||
rows.append(row)
|
||||
except ValueError:
|
||||
# This should not happen. If it does, just skip this row.
|
||||
# It indicates that DomainInformation.domain is None.
|
||||
logger.error("csv_export -> Error when parsing row, domain was None")
|
||||
continue
|
||||
|
||||
if should_write_header:
|
||||
write_header(writer, columns)
|
||||
|
||||
writer.writerows(rows)
|
||||
|
||||
def export_data_type_to_csv(csv_file):
|
||||
"""All domains report with extra columns"""
|
||||
|
|
@ -222,7 +293,6 @@ def export_data_type_to_csv(csv_file):
|
|||
}
|
||||
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True)
|
||||
|
||||
|
||||
def export_data_full_to_csv(csv_file):
|
||||
"""All domains report"""
|
||||
|
||||
|
|
@ -253,7 +323,6 @@ def export_data_full_to_csv(csv_file):
|
|||
}
|
||||
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
|
||||
|
||||
|
||||
def export_data_federal_to_csv(csv_file):
|
||||
"""Federal domains report"""
|
||||
|
||||
|
|
@ -285,18 +354,21 @@ def export_data_federal_to_csv(csv_file):
|
|||
}
|
||||
write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True)
|
||||
|
||||
|
||||
def get_default_start_date():
|
||||
# Default to a date that's prior to our first deployment
|
||||
return timezone.make_aware(datetime(2023, 11, 1))
|
||||
|
||||
|
||||
def get_default_end_date():
|
||||
# Default to now()
|
||||
return timezone.now()
|
||||
|
||||
def format_start_date(start_date):
|
||||
return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date()
|
||||
|
||||
def export_data_growth_to_csv(csv_file, start_date, end_date):
|
||||
def format_end_date(end_date):
|
||||
return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date()
|
||||
|
||||
def export_data_domain_growth_to_csv(csv_file, start_date, end_date):
|
||||
"""
|
||||
Growth report:
|
||||
Receive start and end dates from the view, parse them.
|
||||
|
|
@ -305,16 +377,9 @@ def export_data_growth_to_csv(csv_file, start_date, end_date):
|
|||
the start and end dates. Specify sort params for both lists.
|
||||
"""
|
||||
|
||||
start_date_formatted = (
|
||||
timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date()
|
||||
)
|
||||
|
||||
end_date_formatted = (
|
||||
timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date()
|
||||
)
|
||||
|
||||
start_date_formatted = format_start_date(start_date)
|
||||
end_date_formatted = format_end_date(end_date)
|
||||
writer = csv.writer(csv_file)
|
||||
|
||||
# define columns to include in export
|
||||
columns = [
|
||||
"Domain name",
|
||||
|
|
@ -359,3 +424,127 @@ def export_data_growth_to_csv(csv_file, start_date, end_date):
|
|||
get_domain_managers=False,
|
||||
should_write_header=False,
|
||||
)
|
||||
|
||||
def get_sliced_domains(filter_condition):
|
||||
"""
|
||||
"""
|
||||
|
||||
domains = DomainInformation.objects.all().filter(**filter_condition)
|
||||
federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count()
|
||||
interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count()
|
||||
state_or_territory = domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count()
|
||||
tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count()
|
||||
county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count()
|
||||
city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count()
|
||||
special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count()
|
||||
school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count()
|
||||
election_board = domains.filter(is_election_board=True).count()
|
||||
|
||||
return [federal,
|
||||
interstate,
|
||||
state_or_territory,
|
||||
tribal,
|
||||
county,
|
||||
city,
|
||||
special_district,
|
||||
school_district,
|
||||
election_board]
|
||||
|
||||
def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date):
|
||||
"""
|
||||
"""
|
||||
|
||||
start_date_formatted = format_start_date(start_date)
|
||||
end_date_formatted = format_end_date(end_date)
|
||||
writer = csv.writer(csv_file)
|
||||
columns = [
|
||||
"Domain name",
|
||||
"Domain type",
|
||||
]
|
||||
sort_fields = [
|
||||
"domain__name",
|
||||
]
|
||||
|
||||
writer.writerow(["START DATE"])
|
||||
writer.writerow([])
|
||||
|
||||
filter_managed_domains_start_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__created_at__lte": start_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date)
|
||||
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS"])
|
||||
writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"])
|
||||
writer.writerow(managed_domains_sliced_at_start_date)
|
||||
writer.writerow([])
|
||||
|
||||
write_csv(writer, columns, sort_fields, filter_managed_domains_start_date, get_domain_managers=True, should_write_header=True)
|
||||
writer.writerow([])
|
||||
|
||||
filter_unmanaged_domains_start_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": start_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date)
|
||||
|
||||
writer.writerow(["UNMANAGED DOMAINS COUNTS"])
|
||||
writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"])
|
||||
writer.writerow(unmanaged_domains_sliced_at_start_date)
|
||||
writer.writerow([])
|
||||
write_csv(writer, columns, sort_fields, filter_unmanaged_domains_start_date, get_domain_managers=True, should_write_header=True)
|
||||
writer.writerow([])
|
||||
|
||||
writer.writerow(["END DATE"])
|
||||
writer.writerow([])
|
||||
|
||||
filter_managed_domains_end_date = {
|
||||
"domain__permissions__isnull": False,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date)
|
||||
|
||||
writer.writerow(["MANAGED DOMAINS COUNTS"])
|
||||
writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"])
|
||||
writer.writerow(managed_domains_sliced_at_end_date)
|
||||
writer.writerow([])
|
||||
|
||||
write_csv(writer, columns, sort_fields, filter_managed_domains_end_date, get_domain_managers=True, should_write_header=True)
|
||||
writer.writerow([])
|
||||
|
||||
filter_unmanaged_domains_end_date = {
|
||||
"domain__permissions__isnull": True,
|
||||
"domain__first_ready__lte": end_date_formatted,
|
||||
}
|
||||
unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date)
|
||||
|
||||
writer.writerow(["UNMANAGED DOMAINS COUNTS"])
|
||||
writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"])
|
||||
writer.writerow(unmanaged_domains_sliced_at_end_date)
|
||||
writer.writerow([])
|
||||
|
||||
write_csv(writer, columns, sort_fields, filter_unmanaged_domains_end_date, get_domain_managers=True, should_write_header=True)
|
||||
|
||||
def export_data_requests_to_csv(csv_file, start_date, end_date):
|
||||
"""
|
||||
"""
|
||||
|
||||
start_date_formatted = format_start_date(start_date)
|
||||
end_date_formatted = format_end_date(end_date)
|
||||
writer = csv.writer(csv_file)
|
||||
# define columns to include in export
|
||||
columns = [
|
||||
"Requested domain",
|
||||
"Organization type",
|
||||
"Submission date",
|
||||
]
|
||||
sort_fields = [
|
||||
# "domain__name",
|
||||
]
|
||||
filter_condition = {
|
||||
"status": DomainApplication.ApplicationStatus.SUBMITTED,
|
||||
"submission_date__lte": end_date_formatted,
|
||||
"submission_date__gte": start_date_formatted,
|
||||
}
|
||||
|
||||
write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,31 @@ import logging
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class ExportDataType(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# match the CSV example with all the fields
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"'
|
||||
csv_export.export_data_type_to_csv(response)
|
||||
return response
|
||||
|
||||
class ExportData(View):
|
||||
class ExportDataFull(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# Smaller export based on 1
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="current-full.csv"'
|
||||
csv_export.export_data_full_to_csv(response)
|
||||
return response
|
||||
|
||||
class ExportDataFederal(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# Federal only
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = 'attachment; filename="current-federal.csv"'
|
||||
csv_export.export_data_federal_to_csv(response)
|
||||
return response
|
||||
|
||||
class ExportDataDomainGrowth(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# Get start_date and end_date from the request's GET parameters
|
||||
# #999: not needed if we switch to django forms
|
||||
|
|
@ -19,8 +42,35 @@ class ExportData(View):
|
|||
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"'
|
||||
# For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use
|
||||
# For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use
|
||||
# in context to display this data in the template.
|
||||
csv_export.export_data_growth_to_csv(response, start_date, end_date)
|
||||
csv_export.export_data_domain_growth_to_csv(response, start_date, end_date)
|
||||
|
||||
return response
|
||||
|
||||
class ExportDataManagedVsUnmanaged(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# Get start_date and end_date from the request's GET parameters
|
||||
# #999: not needed if we switch to django forms
|
||||
start_date = request.GET.get("start_date", "")
|
||||
end_date = request.GET.get("end_date", "")
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"'
|
||||
csv_export.export_data_managed_vs_unamanaged_domains(response, start_date, end_date)
|
||||
|
||||
return response
|
||||
|
||||
class ExportDataRequests(View):
|
||||
def get(self, request, *args, **kwargs):
|
||||
# Get start_date and end_date from the request's GET parameters
|
||||
# #999: not needed if we switch to django forms
|
||||
start_date = request.GET.get("start_date", "")
|
||||
end_date = request.GET.get("end_date", "")
|
||||
|
||||
response = HttpResponse(content_type="text/csv")
|
||||
response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"'
|
||||
# For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use
|
||||
# in context to display this data in the template.
|
||||
csv_export.export_data_requests_to_csv(response, start_date, end_date)
|
||||
|
||||
return response
|
||||
Loading…
Add table
Add a link
Reference in a new issue