From 5723cb1fea490d885778fba7cff220d2bf0cfad8 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Tue, 12 Dec 2023 12:30:39 -0600 Subject: [PATCH 01/92] WIP analytics page in Admin --- src/registrar/admin.py | 59 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 12 ++++ 2 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 src/registrar/templates/admin/analytics.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c5f5be276..2156aeeb1 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,15 +1,17 @@ import logging +import datetime + from django import forms -from django.db.models.functions import Concat +from django.db.models.functions import Concat, Avg, F from django.http import HttpResponse -from django.shortcuts import redirect +from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType -from django.http.response import HttpResponseRedirect -from django.urls import reverse +from django.http.response import HttpResponse, HttpResponseRedirect +from django.urls import path, reverse from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.domain import Domain from registrar.models.user import User @@ -353,6 +355,39 @@ class MyUserAdmin(BaseUserAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] + def get_urls(self): + urlpatterns = super().get_urls() + + my_urls = [ + path( + "analytics/", + self.admin_site.admin_view(self.user_analytics), + name="user_analytics", + ), + ] + + return my_urls + urlpatterns + + def user_analytics(self, request): + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + context = dict( + **self.admin_site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + ), + ) + return render(request, "admin/analytics.html", context) + # Let's define First group # (which should in theory be the ONLY group) def group(self, obj): @@ -1095,8 +1130,6 @@ class DomainAdmin(ListHeaderAdmin): return response def get_urls(self): - from django.urls import path - urlpatterns = super().get_urls() # Used to extrapolate a path name, for instance @@ -1178,9 +1211,11 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - "Error deleting this Domain: " - f"Can't switch from state '{obj.state}' to 'deleted'" - ", must be either 'dns_needed' or 'on_hold'", + ( + "Error deleting this Domain: " + f"Can't switch from state '{obj.state}' to 'deleted'" + ", must be either 'dns_needed' or 'on_hold'" + ), messages.ERROR, ) except Exception: @@ -1192,7 +1227,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("Domain %s has been deleted. Thanks!") % obj.name, + "Domain %s has been deleted. Thanks!" % obj.name, ) return HttpResponseRedirect(".") @@ -1234,7 +1269,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("%s is in client hold. This domain is no longer accessible on the public internet.") % obj.name, + "%s is in client hold. This domain is no longer accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") @@ -1263,7 +1298,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("%s is ready. This domain is accessible on the public internet.") % obj.name, + "%s is ready. This domain is accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html new file mode 100644 index 000000000..2906af223 --- /dev/null +++ b/src/registrar/templates/admin/analytics.html @@ -0,0 +1,12 @@ +{% extends "admin/base_site.html" %} + +{% block content_title %}Registrar Analytics{% endblock %} + +{% block content %} + +{% endblock %} From 348d77726035f73e4cd817429e741b4439aa39f0 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Mon, 5 Feb 2024 13:22:17 -0500 Subject: [PATCH 02/92] fix django function imports --- src/registrar/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 2156aeeb1..aa9795eac 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2,7 +2,8 @@ import logging import datetime from django import forms -from django.db.models.functions import Concat, Avg, F +from django.db.models import Avg, F +from django.db.models.functions import Concat from django.http import HttpResponse from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions From 56cd0b6d156f0270f78daf8150e8d79fee1a1b33 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 27 Feb 2024 18:43:59 -0500 Subject: [PATCH 03/92] Gather all existing reports on analytics page --- src/registrar/admin.py | 90 +++++++++---------- src/registrar/assets/sass/_theme/_admin.scss | 3 +- src/registrar/templates/admin/analytics.html | 63 +++++++++++-- src/registrar/templates/admin/index.html | 33 ------- .../django/admin/domain_change_list.html | 23 ----- 5 files changed, 99 insertions(+), 113 deletions(-) delete mode 100644 src/registrar/templates/admin/index.html delete mode 100644 src/registrar/templates/django/admin/domain_change_list.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index aa9795eac..94b8e3dfd 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -357,17 +357,58 @@ class MyUserAdmin(BaseUserAdmin): ordering = ["first_name", "last_name", "email"] def get_urls(self): + """Map a new page in admin for analytics.""" urlpatterns = super().get_urls() + # Used to extrapolate a path name, for instance + # name="{app_label}_{model_name}_export_data_type" + domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name + my_urls = [ path( "analytics/", self.admin_site.admin_view(self.user_analytics), name="user_analytics", ), + path( + "export_data_type/", + self.export_data_type, + name="%s_%s_export_data_type" % domain_path_meta, + ), + path( + "export_data_full/", + self.export_data_full, + name="%s_%s_export_data_full" % domain_path_meta, + ), + path( + "export_data_federal/", + self.export_data_federal, + name="%s_%s_export_data_federal" % domain_path_meta, + ), ] return my_urls + urlpatterns + + def export_data_type(self, request): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + + def export_data_full(self, request): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + + def export_data_federal(self, request): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response def user_analytics(self, request): last_30_days_applications = models.DomainApplication.objects.filter( @@ -1103,60 +1144,11 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def get_urls(self): - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - info = self.model._meta.app_label, self.model._meta.model_name - - my_url = [ - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % info, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % info, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % info, - ), - ] - - return my_url + urlpatterns - def response_change(self, request, obj): # Create dictionary of action functions ACTION_FUNCTIONS = { diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 760c4f13a..04dceef08 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -112,7 +112,8 @@ html[data-theme="light"] { .change-list .usa-table thead th, body.dashboard, body.change-list, - body.change-form { + body.change-form, + .analytics { color: var(--body-fg); } } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2906af223..82081d629 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,12 +1,61 @@ {% extends "admin/base_site.html" %} -{% block content_title %}Registrar Analytics{% endblock %} + + +{% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} -
    -
  • User Count: {{ data.user_count }}
    • -
  • Domain Count: {{ data.domain_count }}
    • -
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
    • -
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
    • -
+ + {% block object-tools %} + + {% endblock %} + +
+
+

At a glance

+
+
    +
  • User Count: {{ data.user_count }}
    • +
  • Domain Count: {{ data.domain_count }}
    • +
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
    • +
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
    • +
+
+
+ +
+

Domain growth

+
+ {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + The challenge is in the path definition in urls. It does NOT like admin/export_data/ + + See the commit "Review for ticket #999" + {% endcomment %} +
+
+ + +
+
+ + +
+ +
+
+
+ +
{% endblock %} diff --git a/src/registrar/templates/admin/index.html b/src/registrar/templates/admin/index.html deleted file mode 100644 index 04601ef32..000000000 --- a/src/registrar/templates/admin/index.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends "admin/index.html" %} - -{% block content %} -
- {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} -
-

Reports

-

Domain growth report

- - {% comment %} - Inputs of type date suck for accessibility. - We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. Itdoes NOT like admin/export_data/ - - See the commit "Review for ticket #999" - {% endcomment %} - -
-
- - -
-
- - -
- - -
- -
-
-{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html deleted file mode 100644 index 22df74685..000000000 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "admin/change_list.html" %} - -{% block object-tools %} - - -{% endblock %} \ No newline at end of file From de17d686e3f73234c81915dabab88961e4f37a55 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:30:23 -0700 Subject: [PATCH 04/92] Add basic on hold modal --- src/registrar/assets/js/get-gov-admin.js | 2 +- .../django/admin/domain_change_form.html | 84 +++++++++++++++++-- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ff73acb65..096f7a626 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -29,7 +29,7 @@ function openInNewTab(el, removeAttribute = false){ */ (function (){ function createPhantomModalFormButtons(){ - let submitButtons = document.querySelectorAll('.usa-modal button[type="submit"]'); + let submitButtons = document.querySelectorAll('.usa-modal button[type="submit"].dja-form-placeholder'); form = document.querySelector("form") submitButtons.forEach((button) => { diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 67c5ac291..e38583eb8 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -11,18 +11,15 @@
{% if original.state != original.State.DELETED %} - + Extend expiration date | {% endif %} {% if original.state == original.State.READY %} - + + Place hold + {% elif original.state == original.State.ON_HOLD %} {% endif %} @@ -52,6 +49,8 @@ In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions of the application, so this means that it will briefly "populate", causing unintended visual effects. {% endcomment %} + + {# Create a modal for the _extend_expiration_date button #}
- +
  • @@ -114,5 +113,74 @@
+ + {# Create a modal for the _on_hold button #} +
+
+
+

+ Are you sure you want to place this domain on hold? +

+
+

+ When a domain is on hold: +

+
    +
  • The domain (and any subdomains) won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain will still appear in the registrar / admin.
    • +
  • Domain managers won’t be able to edit the domain.
    • +
+

+ This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State.ON_HOLD }} +

+
+ +
+
    +
  • + +
    • +
  • + +
    • +
+
+
+ +
+
+ {# Create a modal for when a domain is marked as ineligible #} {{ block.super }} {% endblock %} \ No newline at end of file From f693557f93eeb0d057d46a23ed80f79c4a3ea7b0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:58:40 -0700 Subject: [PATCH 05/92] Add some modals --- src/registrar/admin.py | 2 + .../admin/domain_application_change_form.html | 87 +++++++++++++++++++ .../django/admin/domain_change_form.html | 78 +++++++++++++++-- 3 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 src/registrar/templates/django/admin/domain_application_change_form.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 92e477667..942ae6162 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -931,6 +931,8 @@ class DomainApplicationAdmin(ListHeaderAdmin): if self.value() == "0": return queryset.filter(Q(is_election_board=False) | Q(is_election_board=None)) + change_form_template = "django/admin/domain_application_change_form.html" + # Columns list_display = [ "requested_domain", diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html new file mode 100644 index 000000000..26abbec18 --- /dev/null +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -0,0 +1,87 @@ +{% extends 'admin/change_form.html' %} +{% load i18n static %} + +{% block submit_buttons_bottom %} + {% comment %} + Modals behave very weirdly in django admin. + They tend to "strip out" any injected form elements, leaving only the main form. + In addition, USWDS handles modals by first destroying the element, then repopulating it toward the end of the page. + In effect, this means that the modal is not, and cannot, be surrounded by any form element at compile time. + + The current workaround for this is to use javascript to inject a hidden input, and bind submit of that + element to the click of the confirmation button within this modal. + + This is controlled by the class `dja-form-placeholder` on the button. + + In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions + of the application, so this means that it will briefly "populate", causing unintended visual effects. + {% endcomment %} +{# Create a modal for when a domain is marked as ineligible #} +
+
+
+

+ Are you sure you want to select ineligible status? +

+
+

+ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: +

+
    +
  • They cannot edit the ineligible request or any other pending requests.
    • +
  • They cannot manage any of their approved domains.
    • +
  • They cannot initiate a new domain request.
    • +
+

+ This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State }} +

+
+ +
+
    +
  • + +
    • +
  • + +
    • +
+
+
+ +
+
+{{ block.super }} +{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index e38583eb8..9d7b1d5de 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -27,7 +27,9 @@ | {% endif %} {% if original.state != original.State.DELETED %} - + + Remove from registry + {% endif %} @@ -118,8 +120,8 @@
@@ -131,7 +133,7 @@ When a domain is on hold:

    -
  • The domain (and any subdomains) won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • The domain will still appear in the registrar / admin.
  • Domain managers won’t be able to edit the domain.
@@ -181,6 +183,72 @@
- {# Create a modal for when a domain is marked as ineligible #} + {# Create a modal for the _remove_domain button #} +
+
+
+

+ Are you sure you want to remove this domain from the registry? +

+
+

+ When a domain is removed from the registry: +

+
    +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain will still appear in the registrar / admin.
    • +
  • Domain managers won’t be able to edit the domain.
    • +
+

+ This action cannot be undone. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State.DELETED }} +

+
+ +
+
    +
  • + +
    • +
  • + +
    • +
+
+
+ +
+
{{ block.super }} {% endblock %} \ No newline at end of file From eeff5586d2b0de040ca2ed1614f94db0a9b3d618 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:20:33 -0700 Subject: [PATCH 06/92] Update styling --- src/registrar/assets/sass/_theme/_admin.scss | 9 +++++++++ .../django/admin/domain_change_form.html | 18 +++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index b57c6a015..46e4a10d4 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -302,3 +302,12 @@ input.admin-confirm-button { display: contents !important; } } + +.django-admin-modal .usa-prose ul > li { + list-style-type: inherit; + // Styling based off of the

styling in django admin + line-height: 1.5; + margin-bottom: 0; + margin-top: 0; + max-width: 68ex; +} diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 9d7b1d5de..61007c3d4 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -54,7 +54,7 @@ {# Create a modal for the _extend_expiration_date button #}

+ Domain: {{ original.name }} {# Acts as a
#}

- Domain: {{ original.name }} New status: {{ original.State.ON_HOLD }}

@@ -185,7 +185,7 @@
{# Create a modal for the _remove_domain button #}
    -
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • -
  • The domain will still appear in the registrar / admin.
    • -
  • Domain managers won’t be able to edit the domain.
    • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain will still appear in the registrar / admin.
    • +
  • Domain managers won’t be able to edit the domain.

This action cannot be undone.

+ Domain: {{ original.name }} {# Acts as a
#}

- Domain: {{ original.name }} New status: {{ original.State.DELETED }}

@@ -251,4 +251,4 @@ {{ block.super }} -{% endblock %} \ No newline at end of file +{% endblock %} From 7d65b2cf5cfea3a816e5fc44902f67a7d5993a4d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:15:21 -0700 Subject: [PATCH 07/92] JS changes --- src/registrar/assets/js/get-gov-admin.js | 66 ++++++++- .../admin/domain_application_change_form.html | 135 ++++++++++-------- 2 files changed, 135 insertions(+), 66 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 096f7a626..47351a608 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -35,14 +35,20 @@ function openInNewTab(el, removeAttribute = false){ let input = document.createElement("input"); input.type = "submit"; - input.name = button.name; - input.value = button.value; + + if(button.name){ + input.name = button.name; + } + + if(button.value){ + input.value = button.value; + } + input.style.display = "none" // Add the hidden input to the form form.appendChild(input); button.addEventListener("click", () => { - console.log("clicking") input.click(); }) }) @@ -50,6 +56,60 @@ function openInNewTab(el, removeAttribute = false){ createPhantomModalFormButtons(); })(); + +/** An IIFE for DomainApplication to hook a modal to a dropdown option. + * This intentionally does not interact with createPhantomModalFormButtons() +*/ +(function (){ + function displayModalOnDropdownClick(){ + // Grab the invisible element that will hook to the modal. + // This doesn't technically need to be done with one, but this is simpler to manage. + let linkClickedDisplaysModal = document.getElementById("invisible-ineligible-modal-toggler") + let statusDropdown = document.getElementById("id_status") + + // If these exist all at the same time, we're on the right page + if (linkClickedDisplaysModal && statusDropdown && statusDropdown.value){ + // Store the previous value in the event the user cancels. + // We only need to do this if we're on the correct page. + let previousValue = statusDropdown.value; + // Because the modal button does not have the class "dja-form-placeholder", + // it will not be affected by the createPhantomModalFormButtons() function. + let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); + if (cancelButton){ + console.log(`This is the previous val: ${previousValue}`) + cancelButton.addEventListener('click', function() { + // Revert the dropdown to its previous value + statusDropdown.value = previousValue; + }); + + // Add a change event listener to the dropdown. + statusDropdown.addEventListener('change', function() { + // Check if "Ineligible" is selected + if (this.value && this.value.toLowerCase() === "ineligible") { + // Display the modal. + linkClickedDisplaysModal.click() + } + + // Update previousValue if another option is selected and confirmed + previousValue = this.value; + console.log(`This is the previous val NOW: ${previousValue}`) + }); + + } else{ + console.error("displayModalOnDropdownClick() -> No cancel button defined.") + } + + } + } + + // Adds event listeners on the confirm and cancel modal buttons + function handleModalButtons(){ + + } + + displayModalOnDropdownClick(); +})(); + /** An IIFE for pages in DjangoAdmin which may need custom JS implementation. * Currently only appends target="_blank" to the domain_form object, * but this can be expanded. diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index 26abbec18..f6380fb82 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -1,6 +1,12 @@ {% extends 'admin/change_form.html' %} {% load i18n static %} +{% block field_sets %} + {# Create an invisible tag so that we can use a click event to toggle the modal. #} + + {{ block.super }} +{% endblock %} + {% block submit_buttons_bottom %} {% comment %} Modals behave very weirdly in django admin. @@ -16,72 +22,75 @@ In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions of the application, so this means that it will briefly "populate", causing unintended visual effects. {% endcomment %} -{# Create a modal for when a domain is marked as ineligible #} -
-
-
-

- Are you sure you want to select ineligible status? -

-
-

- When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: -

-
    -
  • They cannot edit the ineligible request or any other pending requests.
    • -
  • They cannot manage any of their approved domains.
    • -
  • They cannot initiate a new domain request.
    • -
-

- This action can be reversed, if needed. -

-

- {# Acts as a
#} -

- Domain: {{ original.name }} - New status: {{ original.State }} -

-
+ {# Create a modal for when a domain is marked as ineligible #} +
+
+
+

+ Are you sure you want to select ineligible status? +

+
+

+ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: +

+
    +
  • They cannot edit the ineligible request or any other pending requests.
    • +
  • They cannot manage any of their approved domains.
    • +
  • They cannot initiate a new domain request.
    • +
+

+ The restrictions will not take effect until you “save” the changes for this domain request. + This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State }} +

+
-
-
    -
  • - -
    • -
  • - -
    • -
+
+
    +
  • + +
    • +
  • + +
    • +
+
+
- -
{{ block.super }} {% endblock %} \ No newline at end of file From 7a7000cf9d358930280585cdb74c8f366ba815bd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:31:08 -0700 Subject: [PATCH 08/92] Fix bug --- src/registrar/assets/js/get-gov-admin.js | 6 ------ .../django/admin/domain_application_change_form.html | 6 +++--- .../templates/django/admin/domain_change_form.html | 6 +++--- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 47351a608..ebcca16d7 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -91,7 +91,6 @@ function openInNewTab(el, removeAttribute = false){ } // Update previousValue if another option is selected and confirmed - previousValue = this.value; console.log(`This is the previous val NOW: ${previousValue}`) }); @@ -102,11 +101,6 @@ function openInNewTab(el, removeAttribute = false){ } } - // Adds event listeners on the confirm and cancel modal buttons - function handleModalButtons(){ - - } - displayModalOnDropdownClick(); })(); diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index f6380fb82..6e6ab3723 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -39,9 +39,9 @@ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows:

    -
  • They cannot edit the ineligible request or any other pending requests.
    • -
  • They cannot manage any of their approved domains.
    • -
  • They cannot initiate a new domain request.
    • +
  • They cannot edit the ineligible request or any other pending requests.
    • +
  • They cannot manage any of their approved domains.
    • +
  • They cannot initiate a new domain request.

The restrictions will not take effect until you “save” the changes for this domain request. diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 61007c3d4..393983e32 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -133,9 +133,9 @@ When a domain is on hold:

    -
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • -
  • The domain will still appear in the registrar / admin.
    • -
  • Domain managers won’t be able to edit the domain.
    • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain will still appear in the registrar / admin.
    • +
  • Domain managers won’t be able to edit the domain.

This action can be reversed, if needed. From 0a174e5d29b9625c0d2454475b67dc8519656541 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 13:58:33 -0500 Subject: [PATCH 09/92] Reports, chart wip --- src/registrar/admin.py | 143 ++++++----- src/registrar/assets/js/get-gov-admin.js | 26 +- src/registrar/assets/sass/_theme/_admin.scss | 17 ++ src/registrar/config/settings.py | 2 +- src/registrar/config/urls.py | 42 +++- src/registrar/signals.py | 2 + src/registrar/templates/admin/analytics.html | 105 ++++++-- src/registrar/templates/admin/base_site.html | 2 + src/registrar/tests/test_admin_views.py | 2 +- src/registrar/tests/test_reports.py | 2 +- src/registrar/utility/csv_export.py | 237 +++++++++++++++++-- src/registrar/views/admin_views.py | 58 ++++- 12 files changed, 498 insertions(+), 140 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 34270584a..9bc77b029 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -27,6 +27,7 @@ from django_fsm import TransitionNotAllowed # type: ignore from django.utils.safestring import mark_safe from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from django.utils.translation import gettext_lazy as _ @@ -363,6 +364,75 @@ class UserContactInline(admin.StackedInline): model = models.Contact +def user_analytics(request): + + end_date = datetime.datetime.today() + start_date = datetime.datetime.today() - datetime.timedelta(days=30) + + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=start_date + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] + + logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # get number of ready domains, counts by org type and election office + # add to context + + # get number of submitted request counts by org type and election office + # add to context + + context = dict( + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ), + ) + return render(request, "admin/analytics.html", context) + class MyUserAdmin(BaseUserAdmin): """Custom user admin class to use our inlines.""" @@ -464,79 +534,6 @@ class MyUserAdmin(BaseUserAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] - def get_urls(self): - """Map a new page in admin for analytics.""" - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name - - my_urls = [ - path( - "analytics/", - self.admin_site.admin_view(self.user_analytics), - name="user_analytics", - ), - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % domain_path_meta, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % domain_path_meta, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % domain_path_meta, - ), - ] - - return my_urls + urlpatterns - - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def user_analytics(self, request): - last_30_days_applications = models.DomainApplication.objects.filter( - created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) - ) - avg_approval_time = last_30_days_applications.annotate( - approval_time=F("approved_domain__created_at") - F("created_at") - ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) - context = dict( - **self.admin_site.each_context(request), - data=dict( - user_count=models.User.objects.all().count(), - domain_count=models.Domain.objects.all().count(), - applications_last_30_days=last_30_days_applications.count(), - average_application_approval_time_last_30_days=avg_approval_time, - ), - ) - return render(request, "admin/analytics.html", context) def get_search_results(self, request, queryset, search_term): """ Override for get_search_results. This affects any upstream model using autocomplete_fields, diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ff73acb65..618cc284c 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -322,23 +322,25 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, // Default the value of the end date input field to the current date let endDateInput =document.getElementById('end'); - let exportGrowthReportButton = document.getElementById('exportLink'); + let exportButtons = document.querySelectorAll('.exportLink'); - if (exportGrowthReportButton) { + if (exportButtons.length > 0) { startDateInput.value = currentDate; endDateInput.value = currentDate; - exportGrowthReportButton.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = document.getElementById('exportLink').dataset.exportUrl; + exportButtons.forEach((btn) => { + btn.addEventListener('click', function() { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); }); } diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 0d232ff41..c74daf678 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -303,3 +303,20 @@ input.admin-confirm-button { display: contents !important; } } + +.usa-button-group { + margin-left: -0.25rem!important; + padding-left: 0!important; + .usa-button-group__item { + list-style-type: none; + line-height: normal; + } + .button { + display: inline-block; + padding: 10px 8px; + line-height: normal; + } + .usa-icon { + top: 2px; + } +} diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index bb8e22ad7..56f3c2090 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -330,7 +330,7 @@ CSP_FORM_ACTION = allowed_sources # Google analytics requires that we relax our otherwise # strict CSP by allowing scripts to run from their domain # and inline with a nonce, as well as allowing connections back to their domain -CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] +CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 4bd7b4baf..a9fee650e 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,9 +9,8 @@ from django.urls import include, path from django.views.generic import RedirectView from registrar import views - -from registrar.views.admin_views import ExportData - +from registrar.admin import user_analytics +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from registrar.views.application import Step from registrar.views.utility import always_404 @@ -52,7 +51,42 @@ urlpatterns = [ "admin/logout/", RedirectView.as_view(pattern_name="logout", permanent=False), ), - path("export_data/", ExportData.as_view(), name="admin_export_data"), + path( + "admin/analytics/export_data_type/", + ExportDataType.as_view(), + name="export_data_type", + ), + path( + "admin/analytics/export_data_full/", + ExportDataFull.as_view(), + name="export_data_full", + ), + path( + "admin/analytics/export_data_federal/", + ExportDataFederal.as_view(), + name="export_data_federal", + ), + path( + "admin/analytics/export_domain_growth/", + ExportDataDomainGrowth.as_view(), + name="export_domain_growth", + ), + path( + "admin/analytics/export_managed_unmanaged/", + ExportDataManagedVsUnmanaged.as_view(), + name="export_managed_unmanaged", + ), + path( + "admin/analytics/export_requests/", + ExportDataRequests.as_view(), + name="export_requests", + ), + path( + "admin/analytics/", + admin.site.admin_view(user_analytics), + name="user_analytics", + ), + path("admin/", admin.site.urls), path( "application//edit/", diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 4e7768ef4..ef09e605b 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,6 +27,7 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") + logger.info(f'in handle_profile first {instance}') is_new_user = kwargs.get("created", False) @@ -36,6 +37,7 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact + logger.info(f'inside no matching contacts for first {first_name} last {last_name} email {email}') Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 82081d629..f65aa77cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,25 +1,11 @@ {% extends "admin/base_site.html" %} - +{% load static %} {% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} - {% block object-tools %} - - {% endblock %} -

At a glance

@@ -34,17 +20,46 @@
-

Domain growth

+

Current domains

+
+ +
+
+ +
+

Growth reports

{% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_data/ + The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ See the commit "Review for ticket #999" {% endcomment %} -
-
+
+
@@ -52,8 +67,58 @@
-
+ +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+ +
+
+ +
+
+ +
+
+
diff --git a/src/registrar/templates/admin/base_site.html b/src/registrar/templates/admin/base_site.html index 73e9ba1f0..58843421a 100644 --- a/src/registrar/templates/admin/base_site.html +++ b/src/registrar/templates/admin/base_site.html @@ -20,7 +20,9 @@ > + + {% endblock %} {% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index aa150d55c..e55175db9 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -26,7 +26,7 @@ class TestViews(TestCase): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 011c60b93..c00c2b221 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -542,7 +542,7 @@ class ExportDataTest(MockEppLib): are pulled when the growth report conditions are applied to export_domains_to_writed. Test that ready domains are sorted by first_ready/deleted dates first, names second. - We considered testing export_data_growth_to_csv which calls write_body + We considered testing export_data_domain_growth_to_csv which calls write_body and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 90e80f551..1764536b5 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,6 +2,7 @@ import csv import logging from datetime import datetime from registrar.models.domain import Domain +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from django.utils import timezone from django.core.paginator import Paginator @@ -19,10 +20,8 @@ def write_header(writer, columns): Receives params from the parent methods and outputs a CSV with a header row. Works with write_header as long as the same writer object is passed. """ - writer.writerow(columns) - def get_domain_infos(filter_condition, sort_fields): domain_infos = ( DomainInformation.objects.select_related("domain", "authorizing_official") @@ -43,7 +42,6 @@ def get_domain_infos(filter_condition, sort_fields): ) return domain_infos_cleaned - def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" @@ -104,7 +102,6 @@ def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None row = [FIELDS.get(column, "") for column in columns] return row - def _get_security_emails(sec_contact_ids): """ Retrieve security contact emails for the given security contact IDs. @@ -126,7 +123,6 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict - def update_columns_with_domain_managers(columns, max_dm_count): """ Update the columns list to include "Domain manager email {#}" headers @@ -135,7 +131,6 @@ def update_columns_with_domain_managers(columns, max_dm_count): for i in range(1, max_dm_count + 1): columns.append(f"Domain manager email {i}") - def write_csv( writer, columns, @@ -148,7 +143,7 @@ def write_csv( Receives params from the parent methods and outputs a CSV with fltered and sorted domains. Works with write_header as longas the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_growth_to_csv calls write_body twice + should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ all_domain_infos = get_domain_infos(filter_condition, sort_fields) @@ -158,15 +153,15 @@ def write_csv( security_emails_dict = _get_security_emails(sec_contact_ids) - # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_domain_infos, 1000) - if get_domain_managers and len(all_domain_infos) > 0: # We want to get the max amont of domain managers an # account has to set the column header dynamically max_dm_count = max(len(domain_info.domain.permissions.all()) for domain_info in all_domain_infos) update_columns_with_domain_managers(columns, max_dm_count) + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_domain_infos, 1000) + for page_num in paginator.page_range: page = paginator.page(page_num) rows = [] @@ -185,6 +180,82 @@ def write_csv( writer.writerows(rows) +def get_domain_requests(filter_condition, sort_fields): + domain_requests = ( + DomainApplication.objects.all() + .filter(**filter_condition) + .order_by(*sort_fields) + ) + + return domain_requests + +def parse_request_row(columns, request: DomainApplication): + """Given a set of columns, generate a new row from cleaned column data""" + + requested_domain_name = 'No requested domain' + + # Domain should never be none when parsing this information + if request.requested_domain is not None: + domain = request.requested_domain + requested_domain_name = domain.name + + domain = request.requested_domain # type: ignore + + if request.federal_type: + request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" + else: + request_type = request.get_organization_type_display() + + # create a dictionary of fields which can be included in output + FIELDS = { + "Requested domain": requested_domain_name, + "Status": request.get_status_display(), + "Organization type": request_type, + "Agency": request.federal_agency, + "Organization name": request.organization_name, + "City": request.city, + "State": request.state_territory, + "AO email": request.authorizing_official.email if request.authorizing_official else " ", + "Security contact email": request, + "Created at": request.created_at, + "Submission date": request.submission_date, + } + + row = [FIELDS.get(column, "") for column in columns] + return row + +def write_requests_csv( + writer, + columns, + sort_fields, + filter_condition, + should_write_header=True, +): + """ + """ + + all_requetsts = get_domain_requests(filter_condition, sort_fields) + + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_requetsts, 1000) + + for page_num in paginator.page_range: + page = paginator.page(page_num) + rows = [] + for request in page.object_list: + try: + row = parse_request_row(columns, request) + rows.append(row) + except ValueError: + # This should not happen. If it does, just skip this row. + # It indicates that DomainInformation.domain is None. + logger.error("csv_export -> Error when parsing row, domain was None") + continue + + if should_write_header: + write_header(writer, columns) + + writer.writerows(rows) def export_data_type_to_csv(csv_file): """All domains report with extra columns""" @@ -222,7 +293,6 @@ def export_data_type_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) - def export_data_full_to_csv(csv_file): """All domains report""" @@ -253,7 +323,6 @@ def export_data_full_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def export_data_federal_to_csv(csv_file): """Federal domains report""" @@ -285,18 +354,21 @@ def export_data_federal_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) - def get_default_end_date(): # Default to now() return timezone.now() +def format_start_date(start_date): + return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() -def export_data_growth_to_csv(csv_file, start_date, end_date): +def format_end_date(end_date): + return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + +def export_data_domain_growth_to_csv(csv_file, start_date, end_date): """ Growth report: Receive start and end dates from the view, parse them. @@ -305,16 +377,9 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): the start and end dates. Specify sort params for both lists. """ - start_date_formatted = ( - timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() - ) - - end_date_formatted = ( - timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - ) - + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) writer = csv.writer(csv_file) - # define columns to include in export columns = [ "Domain name", @@ -359,3 +424,127 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): get_domain_managers=False, should_write_header=False, ) + +def get_sliced_domains(filter_condition): + """ + """ + + domains = DomainInformation.objects.all().filter(**filter_condition) + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domains.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + +def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + writer.writerow(["START DATE"]) + writer.writerow([]) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + writer.writerow(["END DATE"]) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_end_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_end_date, get_domain_managers=True, should_write_header=True) + +def export_data_requests_to_csv(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + # "domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + "submission_date__gte": start_date_formatted, + } + + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f7164663b..4d93aa54b 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -8,9 +8,32 @@ from registrar.utility import csv_export import logging logger = logging.getLogger(__name__) + +class ExportDataType(View): + def get(self, request, *args, **kwargs): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + +class ExportDataFull(View): + def get(self, request, *args, **kwargs): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + +class ExportDataFederal(View): + def get(self, request, *args, **kwargs): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response - -class ExportData(View): +class ExportDataDomainGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters # #999: not needed if we switch to django forms @@ -19,8 +42,35 @@ class ExportData(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use # in context to display this data in the template. - csv_export.export_data_growth_to_csv(response, start_date, end_date) + csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) return response + +class ExportDataManagedVsUnmanaged(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + csv_export.export_data_managed_vs_unamanaged_domains(response, start_date, end_date) + + return response + +class ExportDataRequests(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use + # in context to display this data in the template. + csv_export.export_data_requests_to_csv(response, start_date, end_date) + + return response \ No newline at end of file From 0ef72ad016a975f73922286723e0c5f0ebd3bd29 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:05:19 -0700 Subject: [PATCH 10/92] Add custom desc on delete button --- src/registrar/admin.py | 12 +++++++++++ src/registrar/assets/js/get-gov-admin.js | 1 - .../admin/domain_application_change_form.html | 4 ++-- .../django/admin/domain_change_form.html | 4 ++-- .../admin/domain_delete_confirmation.html | 21 +++++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 src/registrar/templates/django/admin/domain_delete_confirmation.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 942ae6162..fd6ce45a7 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1339,6 +1339,18 @@ class DomainAdmin(ListHeaderAdmin): # Table ordering ordering = ["name"] + def delete_view(self, request, object_id, extra_context=None): + """ + Custom delete_view to perform additional actions or customize the template. + """ + + # Set the delete template to a custom one + self.delete_confirmation_template = "django/admin/domain_delete_confirmation.html" + + response = super().delete_view(request, object_id, extra_context=extra_context) + + return response + def changeform_view(self, request, object_id=None, form_url="", extra_context=None): """Custom changeform implementation to pass in context information""" if extra_context is None: diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ebcca16d7..8ecf2cbee 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -76,7 +76,6 @@ function openInNewTab(el, removeAttribute = false){ // it will not be affected by the createPhantomModalFormButtons() function. let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); if (cancelButton){ - console.log(`This is the previous val: ${previousValue}`) cancelButton.addEventListener('click', function() { // Revert the dropdown to its previous value statusDropdown.value = previousValue; diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index 6e6ab3723..f0e4cfe4f 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -48,10 +48,10 @@ This action can be reversed, if needed.

+ Domain: {{ original.requested_domain.name }} {# Acts as a
#}

- Domain: {{ original.name }} - New status: {{ original.State }} + New status: {{ original.ApplicationStatus.INELIGIBLE|capfirst }}

diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 393983e32..818522c8d 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -144,7 +144,7 @@ Domain: {{ original.name }} {# Acts as a
#}
- New status: {{ original.State.ON_HOLD }} + New status: {{ original.State.ON_HOLD|capfirst }}

@@ -211,7 +211,7 @@ Domain: {{ original.name }} {# Acts as a
#}
- New status: {{ original.State.DELETED }} + New status: {{ original.State.DELETED|capfirst }}

diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html new file mode 100644 index 000000000..793a28c4c --- /dev/null +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -0,0 +1,21 @@ +{% extends 'admin/delete_confirmation.html' %} +{% load i18n static %} + +{% block content %} +{# TODO modify the "Are you sure?" to the text content below.. #} +{% comment %} +

Are you sure you want to remove this domain from the registry?

+{% endcomment %} +

Description

+

When a domain is removed from the registry:

+ +
    +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
    • +
  • The domain will still appear in the registrar / admin.
    • +
  • Domain managers won’t be able to edit the domain.
    • +
+ +

This action cannot be undone.

+ +{{ block.super }} +{% endblock %} From da47cc6f7f661abba3a9dee51dc798ada775b22a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 18:46:45 -0500 Subject: [PATCH 11/92] Add charts to dashboard --- src/registrar/admin.py | 75 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 53 +++++++++++++- src/registrar/utility/csv_export.py | 27 ++++++- 3 files changed, 140 insertions(+), 15 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 9bc77b029..ca6b9bc87 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -384,39 +384,78 @@ def user_analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) + # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] - - logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") - + filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) - managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # Ready and Deleted domains + filter_ready_domains_start_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": start_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_end_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": end_date_formatted, + } + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + - # get number of ready domains, counts by org type and election office - # add to context + # Created and Submitted requests + filter_requests_start_date = { + "submission_date__lte": start_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + + filter_submitted_requests_start_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - # get number of submitted request counts by org type and election office - # add to context + filter_requests_end_date = { + "submission_date__lte": end_date_formatted, + } + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_end_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( **admin.site.each_context(request), @@ -425,10 +464,22 @@ def user_analytics(request): domain_count=models.Domain.objects.all().count(), applications_last_30_days=last_30_days_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_at_end_date=submitted_requests_at_end_date, + ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index f65aa77cf..735386d38 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -7,7 +7,10 @@ {% block content %}
-
+ +
+
+

At a glance

    @@ -19,7 +22,10 @@
-
+
+
+ +

Current domains

    @@ -48,6 +54,9 @@
+
+
+

Growth reports

@@ -119,8 +128,48 @@
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
{% endblock %} diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1764536b5..ce19182b2 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -450,6 +450,31 @@ def get_sliced_domains(filter_condition): school_district, election_board] +def get_sliced_requests(filter_condition): + """ + """ + + domain_requests = DomainApplication.objects.all().filter(**filter_condition) + federal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domain_requests.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): """ """ @@ -470,7 +495,7 @@ def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) From 4b1b1386a8911718887d826950a006853ef46dbd Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 1 Mar 2024 13:44:52 -0500 Subject: [PATCH 12/92] clean up JS, init dates --- src/registrar/assets/js/get-gov-admin.js | 39 -------------------- src/registrar/templates/admin/analytics.html | 32 +++++----------- 2 files changed, 10 insertions(+), 61 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 618cc284c..2c4d4d854 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -307,45 +307,6 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, viewLink.setAttribute('title', viewLink.getAttribute('title-template').replace('selected item', elementText)); } -/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, - * attach the seleted start and end dates to a url that'll trigger the view, and finally - * redirect to that url. -*/ -(function (){ - - // Get the current date in the format YYYY-MM-DD - let currentDate = new Date().toISOString().split('T')[0]; - - // Default the value of the start date input field to the current date - let startDateInput =document.getElementById('start'); - - // Default the value of the end date input field to the current date - let endDateInput =document.getElementById('end'); - - let exportButtons = document.querySelectorAll('.exportLink'); - - if (exportButtons.length > 0) { - startDateInput.value = currentDate; - endDateInput.value = currentDate; - - exportButtons.forEach((btn) => { - btn.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = btn.dataset.exportUrl; - - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; - }); - }); - } - -})(); - /** An IIFE for admin in DjangoAdmin to listen to changes on the domain request * status select amd to show/hide the rejection reason */ diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 735386d38..6eb26307c 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -12,7 +12,7 @@

At a glance

-
+
-

You should probably remove these domains from the registry instead of deleting them.

+

You should probably remove these domains from the registry instead.

This action cannot be undone.

From 7e4dc38b40e7ce92cf9ed426f49a7e4c6ebc345d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:50:03 -0700 Subject: [PATCH 21/92] Change bullet list style --- src/registrar/assets/sass/_theme/_admin.scss | 2 +- .../templates/django/admin/domain_delete_confirmation.html | 2 +- .../django/admin/domain_delete_selected_confirmation.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 9f6db0c46..88675cb32 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -312,7 +312,7 @@ input.admin-confirm-button { max-width: 68ex; } -.django-admin-custom-bullets { +.django-admin-custom-bullets ul > li { // Set list-style-type to inherit without modifying text size list-style-type: inherit; } diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html index 5a9bef5b0..2836c32f7 100644 --- a/src/registrar/templates/django/admin/domain_delete_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -11,7 +11,7 @@

When a domain is deleted:

-
+
  • The domain will no longer appear in the registrar / admin.
  • It will be removed from the registry.
    • diff --git a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html index 3e0a32a4d..6872ea9af 100644 --- a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html @@ -12,7 +12,7 @@

    When a domain is deleted:

    -
    +
    • The domain will no longer appear in the registrar / admin.
    • It will be removed from the registry.
      • From cc1555ab9a76f42455cd995d8cf45daeda1fb785 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:08:52 -0700 Subject: [PATCH 22/92] Unit tests --- src/registrar/assets/sass/_theme/_admin.scss | 5 ----- .../django/admin/domain_delete_confirmation.html | 2 +- .../domain_delete_selected_confirmation.html | 2 +- src/registrar/tests/test_admin.py | 15 ++++++--------- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 88675cb32..cbec4d1f2 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -312,11 +312,6 @@ input.admin-confirm-button { max-width: 68ex; } -.django-admin-custom-bullets ul > li { - // Set list-style-type to inherit without modifying text size - list-style-type: inherit; -} - .usa-summary-box__dhs-color { color: $dhs-blue-70; } diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html index 2836c32f7..5a9bef5b0 100644 --- a/src/registrar/templates/django/admin/domain_delete_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -11,7 +11,7 @@

      When a domain is deleted:

      -
      +
      • The domain will no longer appear in the registrar / admin.
      • It will be removed from the registry.
        • diff --git a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html index 6872ea9af..3e0a32a4d 100644 --- a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html @@ -12,7 +12,7 @@

        When a domain is deleted:

        -
        +
        • The domain will no longer appear in the registrar / admin.
        • It will be removed from the registry.
          • diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 3c5861ee1..f85396f10 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -241,7 +241,7 @@ class TestDomainAdmin(MockEppLib, WebTest): # click the "Manage" link confirmation_page = domain_change_page.click("Delete", index=0) - content_slice = "

          When a domain is removed from the registry:

          " + content_slice = "When a domain is deleted:" self.assertContains(confirmation_page, content_slice) def test_short_org_name_in_domains_list(self): @@ -350,7 +350,7 @@ class TestDomainAdmin(MockEppLib, WebTest): extra_tags="", fail_silently=False, ) - + # The modal should still exist self.assertContains(response, "Are you sure you want to remove this domain from the registry?") self.assertContains(response, "When a domain is removed from the registry:") @@ -364,7 +364,7 @@ class TestDomainAdmin(MockEppLib, WebTest): """ with less_console_noise(): domain = create_ready_domain() - + response = self.app.get(reverse("admin:registrar_domain_change", args=[domain.pk])) # Check the contents of the modal @@ -389,7 +389,7 @@ class TestDomainAdmin(MockEppLib, WebTest): self.assertEqual(response.status_code, 200) self.assertContains(response, domain.name) self.assertContains(response, "Remove hold") - + # The modal should still exist # Check for the header self.assertContains(response, "Are you sure you want to place this domain on hold?") @@ -1180,8 +1180,7 @@ class TestDomainApplicationAdmin(MockEppLib): # Create a mock request request = self.factory.post( - "/admin/registrar/domainapplication/{}/change/".format(application.pk), - follow=True + "/admin/registrar/domainapplication/{}/change/".format(application.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): @@ -1214,7 +1213,6 @@ class TestDomainApplicationAdmin(MockEppLib): self.assertEqual(response.status_code, 200) self.assertContains(response, application.requested_domain.name) - # Check that the modal has the right content # Check for the header self.assertContains(response, "Are you sure you want to select ineligible status?") @@ -1227,8 +1225,7 @@ class TestDomainApplicationAdmin(MockEppLib): # Create a mock request request = self.factory.post( - "/admin/registrar/domainapplication/{}/change/".format(application.pk), - follow=True + "/admin/registrar/domainapplication/{}/change/".format(application.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): # Modify the application's property From 747af8834791b1fa1fee8c0694088eb3964fbef2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:18:59 -0700 Subject: [PATCH 23/92] Add missing unit test + linting --- src/registrar/tests/common.py | 28 ++++++++++++++++++++++++---- src/registrar/tests/test_admin.py | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index ee1ab8b68..1825d38fd 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -97,7 +97,7 @@ def less_console_noise(output_stream=None): class GenericTestHelper(TestCase): """A helper class that contains various helper functions for TestCases""" - def __init__(self, admin, model=None, url=None, user=None, factory=None, **kwargs): + def __init__(self, admin, model=None, url=None, user=None, factory=None, client=None, **kwargs): """ Parameters: admin (ModelAdmin): The Django ModelAdmin instance associated with the model. @@ -112,6 +112,7 @@ class GenericTestHelper(TestCase): self.admin = admin self.model = model self.url = url + self.client = client def assert_table_sorted(self, o_index, sort_fields): """ @@ -147,9 +148,7 @@ class GenericTestHelper(TestCase): dummy_request.user = self.user # Mock a user request - middleware = SessionMiddleware(lambda req: req) - middleware.process_request(dummy_request) - dummy_request.session.save() + dummy_request = self._mock_user_request_for_factory(dummy_request) expected_sort_order = list(self.model.objects.order_by(*sort_fields)) @@ -160,6 +159,27 @@ class GenericTestHelper(TestCase): self.assertEqual(expected_sort_order, returned_sort_order) + def _mock_user_request_for_factory(self, request): + """Adds sessionmiddleware when using factory to associate session information""" + middleware = SessionMiddleware(lambda req: req) + middleware.process_request(request) + request.session.save() + return request + + def get_table_delete_confirmation_page(self, selected_across: str, index: str): + """ + Grabs the response for the delete confirmation page (generated from the actions toolbar). + selected_across and index must both be numbers encoded as str, e.g. "0" rather than 0 + """ + + response = self.client.post( + self.url, + {"action": "delete_selected", "select_across": selected_across, "index": index, "_selected_action": "23"}, + follow=True, + ) + print(f"what is the response? {response}") + return response + class MockUserLogin: def __init__(self, get_response): diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index f85396f10..7e032ff5c 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -61,6 +61,16 @@ class TestDomainAdmin(MockEppLib, WebTest): self.factory = RequestFactory() self.app.set_user(self.superuser.username) self.client.force_login(self.superuser) + + # Contains some test tools + self.test_helper = GenericTestHelper( + factory=self.factory, + user=self.superuser, + admin=self.admin, + url=reverse("admin:registrar_domain_changelist"), + model=Domain, + client=self.client, + ) super().setUp() @skip("TODO for another ticket. This test case is grabbing old db data.") @@ -244,6 +254,21 @@ class TestDomainAdmin(MockEppLib, WebTest): content_slice = "When a domain is deleted:" self.assertContains(confirmation_page, content_slice) + def test_custom_delete_confirmation_page_table(self): + """Tests if we override the delete confirmation page for custom content on the table""" + # Create a ready domain + domain, _ = Domain.objects.get_or_create(name="fake.gov", state=Domain.State.READY) + + # Get the index. The post expects the index to be encoded as a string + index = f"{domain.id}" + + # Simulate selecting a single record, then clicking "Delete selected domains" + response = self.test_helper.get_table_delete_confirmation_page("0", index) + + # Check that our content exists + content_slice = "When a domain is deleted:" + self.assertContains(response, content_slice) + def test_short_org_name_in_domains_list(self): """ Make sure the short name is displaying in admin on the list page From c89d76fb47fb9da7f40d0a29669a2d13020d7cf2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 4 Mar 2024 17:24:53 -0500 Subject: [PATCH 24/92] WIP unit testing --- src/registrar/admin.py | 49 ++- src/registrar/assets/sass/_theme/_admin.scss | 3 + src/registrar/models/domain.py | 2 +- src/registrar/templates/admin/analytics.html | 17 +- src/registrar/templates/admin/app_list.html | 2 +- src/registrar/tests/data/mocks.py | 232 ++++++++++++ src/registrar/tests/test_admin_views.py | 4 +- src/registrar/tests/test_reports.py | 370 ++++++++++++------- src/registrar/utility/csv_export.py | 109 +++--- 9 files changed, 547 insertions(+), 241 deletions(-) create mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 262cebd18..5bf41777b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -374,8 +374,8 @@ def analytics(request): avg_approval_time = last_30_days_approved_applications.annotate( approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) + # Format the timedelta to display only days + avg_approval_time = f"{avg_approval_time.days} days" start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -383,75 +383,69 @@ def analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) - # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - # Ready and Deleted domains filter_ready_domains_start_date = { "domain__state__in": [Domain.State.READY], "domain__first_ready__lte": start_date_formatted, } + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": start_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - - filter_ready_domains_end_date = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - } - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) - filter_deleted_domains_end_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": start_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - - filter_requests_end_date = { - "created_at__lte": end_date_formatted, - } - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) - filter_submitted_requests_end_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( @@ -459,6 +453,7 @@ def analytics(request): data=dict( user_count=models.User.objects.all().count(), domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(), last_30_days_applications=last_30_days_applications.count(), last_30_days_approved_applications=last_30_days_approved_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, @@ -1096,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1448,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] + readonly_fields = ["state", "expiration_date", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index dad88b6a4..29d0e3b2a 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -319,6 +319,9 @@ input.admin-confirm-button { .usa-icon { top: 2px; } + a.button:active, a.button:focus { + text-decoration: none; + } } .module--custom { diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..3b18ac8b6 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper): first_ready = DateField( null=True, - editable=False, + editable=True, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 29faffd3b..380922845 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -16,6 +16,7 @@
          • User Count: {{ data.user_count }}
          • Domain Count: {{ data.domain_count }}
            • +
          • Domains in READY state: {{ data.ready_domain_count }}
          • Domain applications (last 30 days): {{ data.last_30_days_applications }}
          • Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}
          • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
            • @@ -63,8 +64,6 @@ {% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ - See the commit "Review for ticket #999" {% endcomment %}
            @@ -107,7 +106,7 @@
          • -
          -
          - -
          +
          + +
          diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index c96f29a31..4ee2befef 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -71,5 +71,5 @@

          Analytics

          - Dashboard + Dashboard
          \ No newline at end of file diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py new file mode 100644 index 000000000..e6dccb14f --- /dev/null +++ b/src/registrar/tests/data/mocks.py @@ -0,0 +1,232 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation +from registrar.models.domain import Domain +from registrar.models.user_domain_role import UserDomainRole +from registrar.models.public_contact import PublicContact +from registrar.models.user import User +from datetime import date, datetime, timedelta +from django.utils import timezone +from registrar.tests.common import MockEppLib + +class MockDb(MockEppLib): + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + # Test for more than 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + # Test for just 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + # self.domain_request_1, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # requested_domain=self.domain_1.name, + # organization_type="federal", + # federal_agency="World War I Centennial Commission", + # federal_type="executive", + # is_election_board=True + # ) + # self.domain_request_2, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_2, + # organization_type="interstate", + # is_election_board=True + # ) + # self.domain_request_3, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_3, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_4, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_4, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_5, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_5, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_6, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_6, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_7, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_7, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_8, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_8, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_9, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_9, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_10, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_10, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + + def tearDown(self): + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index e55175db9..cc4b3f1c7 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -3,7 +3,7 @@ from django.urls import reverse from registrar.tests.common import create_superuser -class TestViews(TestCase): +class TestAdminViews(TestCase): def setUp(self): self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -26,7 +26,7 @@ class TestViews(TestCase): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index c00c2b221..43efb3128 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -8,9 +8,12 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.common import MockEppLib +from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - write_csv, + format_end_date, + format_start_date, + get_sliced_domains, + write_domains_csv, get_default_start_date, get_default_end_date, ) @@ -231,136 +234,11 @@ class CsvReportsTest(TestCase): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockEppLib): +class ExportDataTest(MockDb): def setUp(self): super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() super().tearDown() def test_export_domains_to_writer_security_emails(self): @@ -403,7 +281,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) @@ -427,7 +305,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_csv(self): + def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" @@ -462,7 +340,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -486,7 +364,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_additional(self): + def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" with less_console_noise(): # Create a CSV file in memory @@ -512,7 +390,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -535,7 +413,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_with_date_filter_pulls_domains_in_range(self): + def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): """Test that domains that are 1. READY and their first_ready dates are in range 2. DELETED and their deleted dates are in range @@ -546,7 +424,7 @@ class ExportDataTest(MockEppLib): and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. - TODO: Simplify is created_at is not needed for the report.""" + TODO: Simplify if created_at is not needed for the report.""" with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -591,7 +469,7 @@ class ExportDataTest(MockEppLib): } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -599,7 +477,7 @@ class ExportDataTest(MockEppLib): get_domain_managers=False, should_write_header=True, ) - write_csv( + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -664,7 +542,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True ) @@ -677,11 +555,11 @@ class ExportDataTest(MockEppLib): expected_content = ( "Domain name,Status,Expiration date,Domain type,Agency," "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager email 1,Domain manager email 2,\n" + "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,info@example.com\n" + ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" ) # Normalize line endings and remove commas, @@ -690,6 +568,210 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + def test_export_data_managed_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + # Call the export functions + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,1\n" + "\n" + "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" + "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_export_data_unmanaged_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # Call the export functions + writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=False, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): + """Test that requests that are + 1. SUBMITTED and their submission_date are in range + are pulled when the growth report conditions are applied to export_requests_to_writed. + Test that requests are sorted by requested domain name. + """ + + pass class HelperFunctions(TestCase): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -704,3 +786,11 @@ class HelperFunctions(TestCase): expected_date = timezone.now() actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) + + def get_sliced_domains(self): + """Should get fitered domains counts sliced by org type and election office.""" + pass + + def test_get_sliced_requests(self): + """Should get fitered requests counts sliced by org type and election office.""" + pass \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index bec5f3835..cbdbfddb3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -25,9 +25,10 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") + DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) + .distinct() ) # Do a mass concat of the first and last name fields for authorizing_official. @@ -44,7 +45,7 @@ def get_domain_infos(filter_condition, sort_fields): return domain_infos_cleaned -def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): +def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" # Domain should never be none when parsing this information @@ -136,7 +137,7 @@ def update_columns_with_domain_managers(columns, max_dm_count): columns.append(f"Domain manager email {i}") -def write_csv( +def write_domains_csv( writer, columns, sort_fields, @@ -145,8 +146,8 @@ def write_csv( should_write_header=True, ): """ - Receives params from the parent methods and outputs a CSV with fltered and sorted domains. - Works with write_header as longas the same writer object is passed. + Receives params from the parent methods and outputs a CSV with filtered and sorted domains. + Works with write_header as long as the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ @@ -172,7 +173,7 @@ def write_csv( rows = [] for domain_info in page.object_list: try: - row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers) + row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers) rows.append(row) except ValueError: # This should not happen. If it does, just skip this row. @@ -188,7 +189,6 @@ def write_csv( def get_requests(filter_condition, sort_fields): requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) - return requests @@ -235,7 +235,8 @@ def write_requests_csv( filter_condition, should_write_header=True, ): - """ """ + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" all_requetsts = get_requests(filter_condition, sort_fields) @@ -295,7 +296,7 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) def export_data_full_to_csv(csv_file): @@ -326,7 +327,7 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def export_data_federal_to_csv(csv_file): @@ -358,7 +359,7 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def get_default_start_date(): @@ -426,8 +427,8 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - write_csv( + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -440,19 +441,19 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): def get_sliced_domains(filter_condition): """Get fitered domains counts sliced by org type and election office.""" - domains = DomainInformation.objects.all().filter(**filter_condition) + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() domains_count = domains.count() - federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() state_or_territory = domains.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = domains.filter(is_election_board=True).count() + ).distinct().count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = domains.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -471,19 +472,19 @@ def get_sliced_domains(filter_condition): def get_sliced_requests(filter_condition): """Get fitered requests counts sliced by org type and election office.""" - requests = DomainApplication.objects.all().filter(**filter_condition) + requests = DomainApplication.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() - interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = requests.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = requests.filter(is_election_board=True).count() + ).distinct().count() + tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = requests.filter(is_election_board=True).distinct().count() return [ requests_count, @@ -500,7 +501,8 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get domains have domain managers for two different dates.""" + """Get counts for domains that have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -512,14 +514,13 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): sort_fields = [ "domain__name", ] - filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"]) + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( [ "Total", @@ -537,16 +538,6 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_managed_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, @@ -571,7 +562,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -582,7 +573,8 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get domains that do not have domain managers for two different dates.""" + """Get counts for domains that do not have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -619,16 +611,6 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, @@ -653,18 +635,23 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_unmanaged_domains_end_date, - get_domain_managers=True, + get_domain_managers=False, should_write_header=True, ) def export_data_requests_growth_to_csv(csv_file, start_date, end_date): - """ """ + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_requests_body SUBMITTED requests that are created between + the start and end dates. Specify sort params. + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -676,7 +663,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date): "Submission date", ] sort_fields = [ - # "domain__name", + "requested_domain__name", ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, From efab77d39f98b3a10c5328af3963c59fd74c5e3b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:58:20 -0700 Subject: [PATCH 25/92] Basic Ao check --- src/registrar/models/domain.py | 1 + src/registrar/views/domain.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..5f17a5332 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -198,6 +198,7 @@ class Domain(TimeStampedModel, DomainHelper): is called in the validate function on the request/domain page throws- RegistryError or InvalidDomainError""" + if not cls.string_could_be_domain(domain): logger.warning("Not a valid domain: %s" % str(domain)) # throw invalid domain error so that it can be caught in diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 72eb65f1e..eec90aeb5 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -10,6 +10,7 @@ import logging from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError +from django.forms import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse @@ -22,6 +23,8 @@ from registrar.models import ( User, UserDomainRole, ) +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact from registrar.utility.enums import DefaultEmail from registrar.utility.errors import ( @@ -225,6 +228,35 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" + # if not self.request.user.is_staff: + + _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) + + current_domain_info = None + if _domain_info.exists() and _domain_info.count() == 1: + current_domain_info = _domain_info.get() + else: + logger.error("Could not update Authorizing Official. No domain info exists, or duplicates exist.") + messages.error(self.request, "Something went wrong when attempting to save.") + return self.form_invalid(form) + + # Determine if the domain is federal or tribal + is_federal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # Get the old and new ao values + old_authorizing_official = form.initial + new_authorizing_official = form.cleaned_data + + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, but it won't always be. + if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: + logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") + messages.error(self.request, "You cannot modify the Authorizing Official.") + + return self.form_invalid(form) + # Set the domain information in the form so that it can be accessible # to associate a new Contact as authorizing official, if new Contact is needed # in the save() method From ff6149a1c42ff89ee9a5b7e8ef6a3010bb97d4df Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 12:51:22 -0500 Subject: [PATCH 26/92] Unit tests plus revert hacks to add data --- src/registrar/admin.py | 4 +- src/registrar/models/domain.py | 2 +- src/registrar/signals.py | 2 - src/registrar/tests/data/mocks.py | 84 ++++------------------ src/registrar/tests/test_reports.py | 106 ++++++++++++++++++++++++++-- src/registrar/utility/csv_export.py | 8 +-- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5bf41777b..78f85f0f9 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1091,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1443,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "deleted"] + readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 3b18ac8b6..449c4c4bb 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper): first_ready = DateField( null=True, - editable=True, + editable=False, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 74dc8a063..4e7768ef4 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,7 +27,6 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") - logger.info(f"in handle_profile first {instance}") is_new_user = kwargs.get("created", False) @@ -37,7 +36,6 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact - logger.info(f"inside no matching contacts for first {first_name} last {last_name} email {email}") Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py index e6dccb14f..25f56f247 100644 --- a/src/registrar/tests/data/mocks.py +++ b/src/registrar/tests/data/mocks.py @@ -1,5 +1,6 @@ from django.test import TestCase from django.contrib.auth import get_user_model +from api.tests.common import less_console_noise from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain @@ -8,7 +9,7 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from datetime import date, datetime, timedelta from django.utils import timezone -from registrar.tests.common import MockEppLib +from registrar.tests.common import MockEppLib, completed_application class MockDb(MockEppLib): def setUp(self): @@ -152,81 +153,22 @@ class MockDb(MockEppLib): user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER ) - # self.domain_request_1, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # requested_domain=self.domain_1.name, - # organization_type="federal", - # federal_agency="World War I Centennial Commission", - # federal_type="executive", - # is_election_board=True - # ) - # self.domain_request_2, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_2, - # organization_type="interstate", - # is_election_board=True - # ) - # self.domain_request_3, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_3, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_4, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_4, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_5, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_5, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_6, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_6, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_7, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_7, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_8, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_8, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_9, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_9, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_10, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_10, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() def tearDown(self): PublicContact.objects.all().delete() Domain.objects.all().delete() DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() User.objects.all().delete() UserDomainRole.objects.all().delete() super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 43efb3128..cc7cc7991 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -2,6 +2,7 @@ import csv import io from django.test import Client, RequestFactory, TestCase from io import StringIO +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain from registrar.models.public_contact import PublicContact @@ -13,9 +14,11 @@ from registrar.utility.csv_export import ( format_end_date, format_start_date, get_sliced_domains, + get_sliced_requests, write_domains_csv, get_default_start_date, get_default_end_date, + write_requests_csv, ) from django.core.management import call_command @@ -27,7 +30,7 @@ import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import less_console_noise +from .common import completed_application, less_console_noise class CsvReportsTest(TestCase): @@ -771,9 +774,58 @@ class ExportDataTest(MockDb): Test that requests are sorted by requested domain name. """ - pass + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + # We use timezone.make_aware to sync to server time a datetime object with the current date + # (using date.today()) and a specific time (using datetime.min.time()). + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) -class HelperFunctions(TestCase): + # Define columns, sort fields, and filter condition + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + "requested_domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + "submission_date__gte": start_date, + } + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name + # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name + expected_content = ( + "Requested domain,Organization type,Submission date\n" + "city3.gov,Federal - Executive,2024-03-05\n" + "city4.gov,Federal - Executive,2024-03-05\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + +class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" def test_get_default_start_date(self): @@ -787,10 +839,52 @@ class HelperFunctions(TestCase): actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) - def get_sliced_domains(self): + def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - pass + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + + expected_content = ( + [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] + ) + + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - pass \ No newline at end of file + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + } + submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + + print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') + + expected_content = ( + [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + ) + + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index cbdbfddb3..e09258022 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -188,7 +188,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) + requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -197,12 +197,8 @@ def parse_request_row(columns, request: DomainApplication): requested_domain_name = "No requested domain" - # Domain should never be none when parsing this information if request.requested_domain is not None: - domain = request.requested_domain - requested_domain_name = domain.name - - domain = request.requested_domain # type: ignore + requested_domain_name = request.requested_domain.name if request.federal_type: request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" From 795a4d71a298d93fdba43407efff9a65def21fb2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:56:31 -0700 Subject: [PATCH 27/92] Check for agency and org name --- src/registrar/views/domain.py | 68 ++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index eec90aeb5..3fa6a96a2 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -10,7 +10,6 @@ import logging from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError -from django.forms import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse @@ -19,12 +18,12 @@ from django.conf import settings from registrar.models import ( Domain, + DomainApplication, + DomainInformation, DomainInvitation, User, UserDomainRole, ) -from registrar.models.domain_application import DomainApplication -from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact from registrar.utility.enums import DefaultEmail from registrar.utility.errors import ( @@ -136,6 +135,20 @@ class DomainFormBaseView(DomainBaseView, FormMixin): # superclass has the redirect return super().form_invalid(form) + + def get_domain_info_from_domain(self) -> DomainInformation | None: + """ + Grabs the underlying domain_info object based off of self.object.name. + Returns None if nothing is found. + """ + _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) + current_domain_info = None + if _domain_info.exists() and _domain_info.count() == 1: + current_domain_info = _domain_info.get() + else: + logger.error("Could get domain_info. No domain info exists, or duplicates exist.") + + return current_domain_info class DomainView(DomainBaseView): @@ -200,6 +213,43 @@ class DomainOrgNameAddressView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the organization name and mailing address.""" + + current_domain_info = self.get_domain_info_from_domain() + if current_domain_info is None: + messages.error(self.request, "Something went wrong when attempting to save.") + return self.form_invalid(form) + + # Get the old and new values to see if a change is occuring + old_org_info = form.initial + new_org_info = form.cleaned_data + + if old_org_info != new_org_info: + + error_message = None + # These actions, aside from the default, should be blocked by the UI, as the field is readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, but it won't always be. + match current_domain_info.organization_type: + case DomainApplication.OrganizationChoices.FEDERAL: + old_fed_agency = old_org_info.get("federal_agency", None) + new_fed_agency = new_org_info.get("federal_agency", None) + if old_fed_agency != new_fed_agency: + error_message = "You cannot modify Federal Agency" + case DomainApplication.OrganizationChoices.TRIBAL: + old_org_name = old_org_info.get("organization_name", None) + new_org_name = new_org_info.get("organization_name", None) + if old_org_name != new_org_name: + error_message = "You cannot modify Organization Name." + case _: + # Do nothing + pass + + # If we encounter an error, forbid this action. + if error_message is not None: + logger.warning(f"User {self.request.user} attempted to change org info on {self.object.name}") + messages.error(self.request, "You cannot modify the Authorizing Official.") + return self.form_invalid(form) + form.save() messages.success(self.request, "The organization information for this domain has been updated.") @@ -229,14 +279,9 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" # if not self.request.user.is_staff: - - _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) - - current_domain_info = None - if _domain_info.exists() and _domain_info.count() == 1: - current_domain_info = _domain_info.get() - else: - logger.error("Could not update Authorizing Official. No domain info exists, or duplicates exist.") + + current_domain_info = self.get_domain_info_from_domain() + if current_domain_info is None: messages.error(self.request, "Something went wrong when attempting to save.") return self.form_invalid(form) @@ -254,7 +299,6 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) # Set the domain information in the form so that it can be accessible From 091e4c900e9853fe2600af2e0fb72e3d1a30ae80 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:28:12 -0500 Subject: [PATCH 28/92] Clean up common mock class and how it's inherited --- src/registrar/tests/common.py | 245 ++++++++++++++++++++++++---- src/registrar/tests/data/mocks.py | 174 -------------------- src/registrar/tests/test_reports.py | 166 +++++-------------- src/registrar/utility/csv_export.py | 4 +- 4 files changed, 251 insertions(+), 338 deletions(-) delete mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index ee1ab8b68..9666d135d 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -13,6 +13,8 @@ from django.contrib.sessions.middleware import SessionMiddleware from django.conf import settings from django.contrib.auth import get_user_model, login from django.utils.timezone import make_aware +from datetime import date, datetime, timedelta +from django.utils import timezone from registrar.models import ( Contact, @@ -35,6 +37,7 @@ from epplibwrapper import ( ErrorCode, responses, ) +from registrar.models.user_domain_role import UserDomainRole from registrar.models.utility.contact_error import ContactError, ContactErrorCodes @@ -470,6 +473,176 @@ class AuditedAdminMockData: application.alternative_domains.add(alt) return application + +class MockDb(TestCase): + """Hardcoded mocks make test case assertions sraightforward.""" + + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + self.end_date = current_date + timedelta(days=2) + self.start_date = current_date - timedelta(days=2) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() + + def tearDown(self): + super().tearDown() + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() def mock_user(): @@ -645,7 +818,7 @@ class MockEppLib(TestCase): self, id, email, - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), pw="thisisnotapassword", ): fake = info.InfoContactResultData( @@ -683,82 +856,82 @@ class MockEppLib(TestCase): mockDataInfoDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meoward.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainAndIPAddress = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoDomainNotSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.subdomainwoip.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataExtensionDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 11, 15), + ex_date=date(2023, 11, 15), ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "123", "123@mail.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithContacts = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -783,7 +956,7 @@ class MockEppLib(TestCase): InfoDomainWithDefaultSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultSec", @@ -798,11 +971,11 @@ class MockEppLib(TestCase): ) mockVerisignDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "defaultVeri", "registrar@dotgov.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "defaultVeri", "registrar@dotgov.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithVerisignSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultVeri", @@ -818,7 +991,7 @@ class MockEppLib(TestCase): InfoDomainWithDefaultTechnicalContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultTech", @@ -843,14 +1016,14 @@ class MockEppLib(TestCase): infoDomainNoContact = fakedEppObject( "security", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["fake.host.com"], ) infoDomainThreeHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.my-nameserver-1.com", @@ -861,43 +1034,43 @@ class MockEppLib(TestCase): infoDomainNoHost = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[], ) infoDomainTwoHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["ns1.my-nameserver-1.com", "ns1.my-nameserver-2.com"], ) mockDataInfoHosts = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="1.2.3.4"), common.Ip(addr="2.3.4.5")], ) mockDataInfoHosts1IP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoHostsNotSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 26, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 26, 19, 45, 35)), addrs=[], ) mockDataInfoHostsSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 27, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 27, 19, 45, 35)), addrs=[], ) - mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35))) + mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35))) addDsData1 = { "keyTag": 1234, "alg": 3, @@ -929,7 +1102,7 @@ class MockEppLib(TestCase): infoDomainHasIP = fakedEppObject( "nameserverwithip.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -954,7 +1127,7 @@ class MockEppLib(TestCase): justNameserver = fakedEppObject( "justnameserver.com", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -977,7 +1150,7 @@ class MockEppLib(TestCase): infoDomainCheckHostIPCombo = fakedEppObject( "nameserversubdomain.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.nameserversubdomain.gov", @@ -987,27 +1160,27 @@ class MockEppLib(TestCase): mockRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockButtonRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2025, 5, 25), + ex_date=date(2025, 5, 25), ) mockDnsNeededRenewedDomainExpDate = fakedEppObject( "fakeneeded.gov", - ex_date=datetime.date(2023, 2, 15), + ex_date=date(2023, 2, 15), ) mockMaximumRenewedDomainExpDate = fakedEppObject( "fakemaximum.gov", - ex_date=datetime.date(2024, 12, 31), + ex_date=date(2024, 12, 31), ) mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", - ex_date=datetime.date(2024, 11, 15), + ex_date=date(2024, 11, 15), ) def _mockDomainName(self, _name, _avail=False): diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py deleted file mode 100644 index 25f56f247..000000000 --- a/src/registrar/tests/data/mocks.py +++ /dev/null @@ -1,174 +0,0 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model -from api.tests.common import less_console_noise -from registrar.models.domain_application import DomainApplication -from registrar.models.domain_information import DomainInformation -from registrar.models.domain import Domain -from registrar.models.user_domain_role import UserDomainRole -from registrar.models.public_contact import PublicContact -from registrar.models.user import User -from datetime import date, datetime, timedelta -from django.utils import timezone -from registrar.tests.common import MockEppLib, completed_application - -class MockDb(MockEppLib): - def setUp(self): - super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - is_election_board=True - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - lebowski_user = get_user_model().objects.create( - username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) - - with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") - self.domain_request_3.submit() - self.domain_request_3.save() - self.domain_request_4.submit() - self.domain_request_4.save() - - def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - DomainApplication.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() - super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cc7cc7991..e6230fadb 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,14 +5,9 @@ from io import StringIO from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain -from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model -from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - format_end_date, - format_start_date, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -30,60 +25,17 @@ import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import completed_application, less_console_noise +from .common import MockDb, MockEppLib, less_console_noise -class CsvReportsTest(TestCase): +class CsvReportsTest(MockDb): """Tests to determine if we are uploading our reports correctly""" def setUp(self): """Create fake domain data""" + super().setUp() self.client = Client(HTTP_HOST="localhost:8080") self.factory = RequestFactory() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - def tearDown(self): - """Delete all faked data""" - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - super().tearDown() @boto3_mocking.patching def test_generate_federal_report(self): @@ -94,6 +46,7 @@ class CsvReportsTest(TestCase): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -114,6 +67,7 @@ class CsvReportsTest(TestCase): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -172,6 +126,7 @@ class CsvReportsTest(TestCase): @boto3_mocking.patching def test_load_federal_report(self): """Tests the get_current_federal api endpoint""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -205,6 +160,7 @@ class CsvReportsTest(TestCase): @boto3_mocking.patching def test_load_full_report(self): """Tests the current-federal api link""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -237,7 +193,7 @@ class CsvReportsTest(TestCase): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockDb): +class ExportDataTest(MockDb, MockEppLib): def setUp(self): super().setUp() @@ -247,6 +203,7 @@ class ExportDataTest(MockDb): def test_export_domains_to_writer_security_emails(self): """Test that export_domains_to_writer returns the expected security email""" + with less_console_noise(): # Add security email information self.domain_1.name = "defaultsecurity.gov" @@ -312,6 +269,7 @@ class ExportDataTest(MockDb): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -369,6 +327,7 @@ class ExportDataTest(MockDb): def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -428,15 +387,11 @@ class ExportDataTest(MockDb): which are hard to mock. TODO: Simplify if created_at is not needed for the report.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) - # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -460,15 +415,15 @@ class ExportDataTest(MockDb): "domain__state__in": [ Domain.State.READY, ], - "domain__first_ready__lte": end_date, - "domain__first_ready__gte": start_date, + "domain__first_ready__lte": self.end_date, + "domain__first_ready__gte": self.start_date, } filter_conditions_for_deleted_domains = { "domain__state__in": [ Domain.State.DELETED, ], - "domain__deleted__lte": end_date, - "domain__deleted__gte": start_date, + "domain__deleted__lte": self.end_date, + "domain__deleted__gte": self.start_date, } # Call the export functions @@ -515,13 +470,13 @@ class ExportDataTest(MockDb): def test_export_domains_to_writer_domain_managers(self): """Test that export_domains_to_writer returns the - expected domain managers""" + expected domain managers.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition - columns = [ "Domain name", "Status", @@ -572,13 +527,13 @@ class ExportDataTest(MockDb): self.assertEqual(csv_content, expected_content) def test_export_data_managed_domains_to_csv(self): - """""" + """Test get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -589,7 +544,7 @@ class ExportDataTest(MockDb): ] filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) # Call the export functions @@ -610,13 +565,11 @@ class ExportDataTest(MockDb): ) writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -634,7 +587,6 @@ class ExportDataTest(MockDb): ) writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -647,9 +599,7 @@ class ExportDataTest(MockDb): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" @@ -663,20 +613,22 @@ class ExportDataTest(MockDb): "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): - """""" + """Test get counts for domains that do not have domain managers for two different dates, + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -687,7 +639,7 @@ class ExportDataTest(MockDb): ] filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) # Call the export functions @@ -708,13 +660,11 @@ class ExportDataTest(MockDb): ) writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -732,7 +682,6 @@ class ExportDataTest(MockDb): ) writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -745,9 +694,7 @@ class ExportDataTest(MockDb): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" @@ -761,10 +708,12 @@ class ExportDataTest(MockDb): "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -778,17 +727,6 @@ class ExportDataTest(MockDb): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - # Define columns, sort fields, and filter condition columns = [ "Requested domain", @@ -800,16 +738,14 @@ class ExportDataTest(MockDb): ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, - "submission_date__gte": start_date, + "submission_date__lte": self.end_date, + "submission_date__gte": self.start_date, } write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) # Reset the CSV file's position to the beginning csv_file.seek(0) - # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( @@ -817,12 +753,12 @@ class ExportDataTest(MockDb): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -841,50 +777,28 @@ class HelperFunctions(MockDb): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] ) - self.assertEqual(managed_domains_sliced_at_end_date, expected_content) - - def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, + "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - - print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') - expected_content = ( [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1bdd3fd82..22467bf6b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -498,7 +498,7 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that have domain managers for two different dates, - get list of domains at end_date.""" + get list of managed domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -570,7 +570,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that do not have domain managers for two different dates, - get list of domains at end_date.""" + get list of unmanaged domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) From 2081f5c56483cf9dadd3f5f71d06204c3055d184 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:41:34 -0500 Subject: [PATCH 29/92] lint --- src/registrar/admin.py | 2 +- src/registrar/tests/common.py | 47 ++++++++++++++----------- src/registrar/tests/test_reports.py | 53 +++++++++++++++-------------- src/registrar/utility/csv_export.py | 46 +++++++++++++++++-------- 4 files changed, 86 insertions(+), 62 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 78f85f0f9..3d6d87367 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -424,7 +424,7 @@ def analytics(request): "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 9666d135d..e6e642918 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -1,4 +1,3 @@ -import datetime import os import logging @@ -473,7 +472,8 @@ class AuditedAdminMockData: application.alternative_domains.add(alt) return application - + + class MockDb(TestCase): """Hardcoded mocks make test case assertions sraightforward.""" @@ -535,69 +535,66 @@ class MockDb(TestCase): organization_type="federal", federal_agency="World War I Centennial Commission", federal_type="executive", - is_election_board=True + is_election_board=True, ) self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True + creator=self.user, domain=self.domain_2, organization_type="interstate", is_election_board=True ) self.domain_information_3, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_3, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_4, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_4, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_5, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_5, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_6, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_6, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_7, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_7, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_8, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_8, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_9, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_9, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_10, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_10, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) meoward_user = get_user_model().objects.create( @@ -625,11 +622,21 @@ class MockDb(TestCase): ) with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_1 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov" + ) + self.domain_request_2 = completed_application( + status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov" + ) + self.domain_request_3 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov" + ) + self.domain_request_4 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov" + ) + self.domain_request_5 = completed_application( + status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov" + ) self.domain_request_3.submit() self.domain_request_3.save() self.domain_request_4.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 55f5c9108..03f792825 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -43,7 +43,7 @@ class CsvReportsTest(MockDb): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -64,7 +64,7 @@ class CsvReportsTest(MockDb): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -525,7 +525,7 @@ class ExportDataTest(MockDb, MockEppLib): def test_export_data_managed_domains_to_csv(self): """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" + get list of managed domains at end_date.""" with less_console_noise(): # Create a CSV file in memory @@ -596,32 +596,34 @@ class ExportDataTest(MockDb, MockEppLib): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" "1,1,0,0,0,0,0,0,0,1\n" "\n" "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -691,26 +693,28 @@ class ExportDataTest(MockDb, MockEppLib): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "UNMANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" "\n" "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -750,14 +754,15 @@ class ExportDataTest(MockDb, MockEppLib): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) + class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -774,28 +779,24 @@ class HelperFunctions(MockDb): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( - [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] - ) + expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - expected_content = ( - [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] - ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file + expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 44e34164d..fdebfef77 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -128,6 +128,7 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict + def write_domains_csv( writer, columns, @@ -253,7 +254,6 @@ def write_requests_csv( logger.error("csv_export -> Error when parsing row, domain was None") continue - if should_write_header: write_header(writer, columns) writer.writerows(rows) @@ -293,7 +293,9 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): @@ -324,7 +326,9 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): @@ -356,7 +360,9 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): @@ -424,7 +430,9 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) write_domains_csv( writer, columns, @@ -442,14 +450,18 @@ def get_sliced_domains(filter_condition): domains_count = domains.count() federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() - state_or_territory = domains.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = domains.filter(is_election_board=True).distinct().count() return [ @@ -473,14 +485,18 @@ def get_sliced_requests(filter_condition): requests_count = requests.count() federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = requests.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = requests.filter(is_election_board=True).distinct().count() return [ From 05533179b67d380e9c69d39c68bc23703ef2e1de Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:51:35 -0500 Subject: [PATCH 30/92] cleanup --- src/registrar/admin.py | 3 ++- src/registrar/templates/admin/analytics.html | 1 - src/registrar/templates/admin/app_list.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3d6d87367..41391f724 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -364,6 +364,7 @@ class UserContactInline(admin.StackedInline): def analytics(request): + """View for the reports page.""" thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) @@ -377,6 +378,7 @@ def analytics(request): # Format the timedelta to display only days avg_approval_time = f"{avg_approval_time.days} days" + # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -427,7 +429,6 @@ def analytics(request): deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) - # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 380922845..72aa244cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,7 +1,6 @@ {% extends "admin/base_site.html" %} {% load static %} - {% block content_title %}

          Registrar Analytics

          {% endblock %} {% block content %} diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index 4ee2befef..dd7e27f33 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -72,4 +72,4 @@

          Analytics

          Dashboard -
          \ No newline at end of file +
          From 11f69454c14869c28a95e6a3fd7a85578200a262 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:53:32 -0500 Subject: [PATCH 31/92] cleanup --- src/registrar/views/admin_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index c3769ad03..04f98a2c4 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -77,7 +77,7 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) @@ -92,7 +92,7 @@ class ExportDataUnmanagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) From 62cf4ecb687077a2c857f53fd2b426c9e3c08b32 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:59:37 -0500 Subject: [PATCH 32/92] lint --- src/registrar/views/admin_views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04f98a2c4..04fcaa6f2 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -76,9 +76,7 @@ class ExportDataManagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) return response @@ -91,9 +89,7 @@ class ExportDataUnmanagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From a9878a00730ca2cadf20872207d86ea74729b64d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:01:27 -0500 Subject: [PATCH 33/92] Charts js --- src/registrar/assets/js/get-gov-reports.js | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/registrar/assets/js/get-gov-reports.js diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js new file mode 100644 index 000000000..e900fabe8 --- /dev/null +++ b/src/registrar/assets/js/get-gov-reports.js @@ -0,0 +1,117 @@ +/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, + * attach the seleted start and end dates to a url that'll trigger the view, and finally + * redirect to that url. + * + * This function also sets the start and end dates to match the url params if they exist +*/ +(function () { + // Function to get URL parameter value by name + function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); + } + + // Get the current date in the format YYYY-MM-DD + let currentDate = new Date().toISOString().split('T')[0]; + + // Default the value of the start date input field to the current date + let startDateInput = document.getElementById('start'); + + // Default the value of the end date input field to the current date + let endDateInput = document.getElementById('end'); + + let exportButtons = document.querySelectorAll('.exportLink'); + + if (exportButtons.length > 0) { + // Check if start and end dates are present in the URL + let urlStartDate = getParameterByName('start_date'); + let urlEndDate = getParameterByName('end_date'); + + // Set input values based on URL parameters or current date + startDateInput.value = urlStartDate || currentDate; + endDateInput.value = urlEndDate || currentDate; + + exportButtons.forEach((btn) => { + btn.addEventListener('click', function () { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); + }); + } + +})(); + +document.addEventListener("DOMContentLoaded", function () { + createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); + createComparativeColumnChart("myChart6", "All requests", "Start Date", "End Date"); +}); + +function createComparativeColumnChart(canvasId, title, labelOne, labelTwo) { + var canvas = document.getElementById(canvasId); + var ctx = canvas.getContext("2d"); + + var listOne = JSON.parse(canvas.getAttribute('data-list-one')); + var listTwo = JSON.parse(canvas.getAttribute('data-list-two')); + + var data = { + labels: ["Total", "Federal", "Interstate", "State/Territory", "Tribal", "County", "City", "Special District", "School District", "Election Board"], + datasets: [ + { + label: labelOne, + backgroundColor: "rgba(255, 99, 132, 0.2)", + borderColor: "rgba(255, 99, 132, 1)", + borderWidth: 1, + data: listOne, + }, + { + label: labelTwo, + backgroundColor: "rgba(75, 192, 192, 0.2)", + borderColor: "rgba(75, 192, 192, 1)", + borderWidth: 1, + data: listTwo, + }, + ], + }; + + var options = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'top', + }, + title: { + display: true, + text: title + } + }, + scales: { + y: { + beginAtZero: true, + }, + }, + }; + + new Chart(ctx, { + type: "bar", + data: data, + options: options, + }); +} \ No newline at end of file From 20161fe7f64c1c0ea215e8c01aa1f3551190f5eb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:18:59 -0700 Subject: [PATCH 34/92] Simplify logic --- src/registrar/forms/domain.py | 75 ++++++++++++++++++++++++++++++----- src/registrar/views/domain.py | 61 +--------------------------- 2 files changed, 65 insertions(+), 71 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 1669774ae..3081cf7c3 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,9 +1,9 @@ """Forms for domain management.""" - +import logging from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator from django.forms import formset_factory - +from registrar.models import DomainApplication from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.utility.errors import ( NameserverError, @@ -23,6 +23,9 @@ from .common import ( import re +logger = logging.getLogger(__name__) + + class DomainAddUserForm(forms.Form): """Form for adding a user to a domain.""" @@ -205,6 +208,13 @@ class ContactForm(forms.ModelForm): "required": "Enter your email address in the required format, like name@example.com." } self.fields["phone"].error_messages["required"] = "Enter your phone number." + self.domainInfo = None + + def set_domain_info(self, domainInfo): + """Set the domain information for the form. + The form instance is associated with the contact itself. In order to access the associated + domain information object, this needs to be set in the form by the view.""" + self.domainInfo = domainInfo class AuthorizingOfficialContactForm(ContactForm): @@ -232,20 +242,32 @@ class AuthorizingOfficialContactForm(ContactForm): self.fields["email"].error_messages = { "required": "Enter an email address in the required format, like name@example.com." } - self.domainInfo = None - - def set_domain_info(self, domainInfo): - """Set the domain information for the form. - The form instance is associated with the contact itself. In order to access the associated - domain information object, this needs to be set in the form by the view.""" - self.domainInfo = domainInfo def save(self, commit=True): - """Override the save() method of the BaseModelForm.""" + """ + Override the save() method of the BaseModelForm. + Used to perform checks on the underlying domain_information object. + If this doesn't exist, we just save as normal. + """ + + # If the underlying Domain doesn't have a domainInfo object, + # just let the default super handle it. + if not self.domainInfo: + return super().save() + + # Determine if the domain is federal or tribal + is_federal = self.domainInfo.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.domainInfo.organization_type == DomainApplication.OrganizationChoices.TRIBAL # Get the Contact object from the db for the Authorizing Official db_ao = Contact.objects.get(id=self.instance.id) - if self.domainInfo and db_ao.has_more_than_one_join("information_authorizing_official"): + + if (is_federal or is_tribal) and self.has_changed(): + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, so lets reserve information for the backend only. + raise ValueError("Authorizing Official cannot be modified for federal or tribal domains.") + elif db_ao.has_more_than_one_join("information_authorizing_official"): # Handle the case where the domain information object is available and the AO Contact # has more than one joined object. # In this case, create a new Contact, and update the new Contact with form data. @@ -254,6 +276,7 @@ class AuthorizingOfficialContactForm(ContactForm): self.domainInfo.authorizing_official = Contact.objects.create(**data) self.domainInfo.save() else: + # If all checks pass, just save normally super().save() @@ -333,6 +356,36 @@ class DomainOrgNameAddressForm(forms.ModelForm): self.fields[field_name].required = True self.fields["state_territory"].widget.attrs.pop("maxlength", None) self.fields["zipcode"].widget.attrs.pop("maxlength", None) + + def save(self, commit=True): + """Override the save() method of the BaseModelForm.""" + if self.has_changed(): + is_federal = self.instance.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.instance.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, so lets reserve information for the backend only. + if is_federal and not self._field_unchanged("federal_agency"): + raise ValueError("federal_agency cannot be modified when the organization_type is federal") + elif is_tribal and not self._field_unchanged("organization_name"): + raise ValueError("organization_name cannot be modified when the organization_type is tribal") + + else: + super().save() + + def _field_unchanged(self, field_name) -> bool: + """ + Checks if a specified field has not changed between the old value + and the new value. + + The old value is grabbed from self.initial. + The new value is grabbed from self.cleaned_data. + """ + old_value = self.initial.get(field_name, None) + new_value = self.cleaned_data.get(field_name, None) + return old_value == new_value + class DomainDnssecForm(forms.Form): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 3fa6a96a2..8598e93fe 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -213,43 +213,6 @@ class DomainOrgNameAddressView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the organization name and mailing address.""" - - current_domain_info = self.get_domain_info_from_domain() - if current_domain_info is None: - messages.error(self.request, "Something went wrong when attempting to save.") - return self.form_invalid(form) - - # Get the old and new values to see if a change is occuring - old_org_info = form.initial - new_org_info = form.cleaned_data - - if old_org_info != new_org_info: - - error_message = None - # These actions, aside from the default, should be blocked by the UI, as the field is readonly. - # If they get past this point, we forbid it this way. - # This could be malicious, but it won't always be. - match current_domain_info.organization_type: - case DomainApplication.OrganizationChoices.FEDERAL: - old_fed_agency = old_org_info.get("federal_agency", None) - new_fed_agency = new_org_info.get("federal_agency", None) - if old_fed_agency != new_fed_agency: - error_message = "You cannot modify Federal Agency" - case DomainApplication.OrganizationChoices.TRIBAL: - old_org_name = old_org_info.get("organization_name", None) - new_org_name = new_org_info.get("organization_name", None) - if old_org_name != new_org_name: - error_message = "You cannot modify Organization Name." - case _: - # Do nothing - pass - - # If we encounter an error, forbid this action. - if error_message is not None: - logger.warning(f"User {self.request.user} attempted to change org info on {self.object.name}") - messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) - form.save() messages.success(self.request, "The organization information for this domain has been updated.") @@ -278,31 +241,9 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" - # if not self.request.user.is_staff: - - current_domain_info = self.get_domain_info_from_domain() - if current_domain_info is None: - messages.error(self.request, "Something went wrong when attempting to save.") - return self.form_invalid(form) - - # Determine if the domain is federal or tribal - is_federal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.FEDERAL - is_tribal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.TRIBAL - - # Get the old and new ao values - old_authorizing_official = form.initial - new_authorizing_official = form.cleaned_data - - # This action should be blocked by the UI, as the text fields are readonly. - # If they get past this point, we forbid it this way. - # This could be malicious, but it won't always be. - if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: - logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") - messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) # Set the domain information in the form so that it can be accessible - # to associate a new Contact as authorizing official, if new Contact is needed + # to associate a new Contact, if a new Contact is needed # in the save() method form.set_domain_info(self.object.domain_info) form.save() From fda2c11fb1a329d1ff03e7108484ccea52947b8d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:33:55 -0500 Subject: [PATCH 35/92] Some accessibility work on charts --- src/registrar/admin.py | 2 + src/registrar/assets/js/get-gov-reports.js | 4 +- src/registrar/templates/admin/analytics.html | 54 +++++++++++++++----- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 41391f724..5b8d67983 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -470,6 +470,8 @@ def analytics(request): submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, requests_sliced_at_end_date=requests_sliced_at_end_date, submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js index e900fabe8..d10cf2dc6 100644 --- a/src/registrar/assets/js/get-gov-reports.js +++ b/src/registrar/assets/js/get-gov-reports.js @@ -55,8 +55,8 @@ })(); document.addEventListener("DOMContentLoaded", function () { - createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); - createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart1", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Unmanaged domains", "Start Date", "End Date"); createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 72aa244cf..da7f25c66 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -115,46 +115,76 @@
          - + > +

          Chart: Managed domains

          +

          {{ data.managed_domains_sliced_at_end_date.0 }} managed domains for {{ data.end_date }}

          +
          - + > +

          Chart: Unanaged domains

          +

          {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

          +
          - + > +

          Chart: Deleted domains

          +

          {{ data.deleted_domains_sliced_at_end_date.0 }} deleted domains for {{ data.end_date }}

          +
          - + > +

          Chart: Ready domains

          +

          {{ data.ready_domains_sliced_at_end_date.0 }} ready domains for {{ data.end_date }}

          +
          - + > +

          Chart: Submitted requests

          +

          {{ data.submitted_requests_sliced_at_end_date.0 }} submitted requests for {{ data.end_date }}

          +
          - + > +

          Chart: All requests

          +

          {{ data.requests_sliced_at_end_date.0 }} requests for {{ data.end_date }}

          +
          From 550bed1de6641e4e319b40e98091e7fc150f0d30 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:52:25 -0700 Subject: [PATCH 36/92] Basic styling / HTML content Needs some work, but this is fine for now --- src/registrar/assets/sass/_theme/_forms.scss | 8 ++++++ src/registrar/forms/domain.py | 27 ++++++++++++++++++- .../domain_authorizing_official.html | 9 ++++++- src/registrar/views/domain.py | 12 +++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_forms.scss b/src/registrar/assets/sass/_theme/_forms.scss index 94407f88d..7ebdaca31 100644 --- a/src/registrar/assets/sass/_theme/_forms.scss +++ b/src/registrar/assets/sass/_theme/_forms.scss @@ -38,3 +38,11 @@ legend.float-left-tablet + button.float-right-tablet { margin-top: 1rem; } } + +/* Custom style for disabled inputs */ +// TODO - UPDATE THIS! +.usa-input:disabled, .usa-select:disabled, .usa-textarea:disabled { + background-color: #f0f0f0; + color: #5b616b; + border-color: #5b616b; +} diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 3081cf7c3..f02b0eea1 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -222,7 +222,7 @@ class AuthorizingOfficialContactForm(ContactForm): JOIN = "authorizing_official" - def __init__(self, *args, **kwargs): + def __init__(self, disable_fields=False, *args, **kwargs): super().__init__(*args, **kwargs) # Overriding bc phone not required in this form @@ -243,6 +243,17 @@ class AuthorizingOfficialContactForm(ContactForm): "required": "Enter an email address in the required format, like name@example.com." } + # TODO - uswds text fields dont have disabled styling?? + # All fields should be disabled if the domain is federal or tribal + if disable_fields: + self._mass_disable_fields() + + def _mass_disable_fields(self): + """Given all available fields, invoke .disabled = True on them""" + for field in self.fields.values(): + field.disabled = True + + def save(self, commit=True): """ Override the save() method of the BaseModelForm. @@ -356,6 +367,20 @@ class DomainOrgNameAddressForm(forms.ModelForm): self.fields[field_name].required = True self.fields["state_territory"].widget.attrs.pop("maxlength", None) self.fields["zipcode"].widget.attrs.pop("maxlength", None) + + is_federal = self.instance.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.instance.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # (Q) Should required = False be set here? + # These fields should not be None. If they are, + # it seems like an analyst should intervene? + + # TODO - maybe consider adding a modal on these fields on hover + # ALSO TODO - uswds text fields dont have disabled styling?? + if is_federal: + self.fields['federal_agency'].disabled = True + elif is_tribal: + self.fields['organization_name'].disabled = True def save(self, commit=True): """Override the save() method of the BaseModelForm.""" diff --git a/src/registrar/templates/domain_authorizing_official.html b/src/registrar/templates/domain_authorizing_official.html index e7fc12a5e..6768f8b68 100644 --- a/src/registrar/templates/domain_authorizing_official.html +++ b/src/registrar/templates/domain_authorizing_official.html @@ -11,6 +11,13 @@

          Your authorizing official is a person within your organization who can authorize domain requests. This person must be in a role of significant, executive responsibility within the organization. Read more about who can serve as an authorizing official.

          + + {% if organization_type == "federal" or organization_type == "tribal" %} +

          + The authorizing official for your organization can’t be updated here. + To suggest an update, email help@get.gov. +

          + {% endif %} {% include "includes/required_fields.html" %} @@ -24,7 +31,7 @@ {% input_with_errors form.title %} {% input_with_errors form.email %} - + - - + {% if organization_type != "federal" and organization_type != "tribal" %} + + {% endif %} + {% endblock %} {# domain_content #} diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 59b5faaa9..8cc3b16a0 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1087,6 +1087,101 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Not igorville") self.assertContains(success_result_page, "Faketown") + + def test_domain_org_name_address_form_tribal(self): + """ + Submitting a change to organization_name is blocked for tribal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + tribal_org_type = DomainInformation.OrganizationChoices.TRIBAL + self.domain_information.organization_type = tribal_org_type + self.domain_information.save() + try: + # Add an org name + self.domain_information.organization_name = "Town of Igorville" + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") + + self.assertEqual(self.domain_information.organization_type, tribal_org_type) + + org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + print(f"what is the org name page? {org_name_page}") + + form = org_name_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["organization_name"][0] + self.assertEqual(organization_name_input.value, "Town of Igorville") + + # Check if the input field is disabled + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["organization_name"] = "Not igorville" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The org name should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "Town of Igorville") + self.assertNotContains(success_result_page, "Not igorville") + + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") + + def test_domain_org_name_address_form_federal(self): + """ + Submitting a change to federal_agency is blocked for federal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + federal_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = federal_org_type + self.domain_information.save() + try: + # Add a federal agency. Defined as a tuple since this list may change order. + self.domain_information.federal_agency = ("AMTRAK", "AMTRAK") + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") + + self.assertEqual(self.domain_information.organization_type, federal_org_type) + + org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + + form = org_name_page.forms[0] + # Check the value of the input field + federal_agency_input = form.fields["federal_agency"][0] + self.assertEqual(federal_agency_input.value, "AMTRAK") + + # Check if the input field is disabled + self.assertTrue("disabled" in federal_agency_input.attrs) + self.assertEqual(federal_agency_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["organization_name"] = "Not igorville" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The org name should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "Town of Igorville") + self.assertNotContains(success_result_page, "Not igorville") + + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") class TestDomainContactInformation(TestDomainOverview): From b50eaf91f623c42bb57a06b49ed43b21c7e07385 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:52:31 -0700 Subject: [PATCH 38/92] Unit tests --- src/registrar/forms/domain.py | 4 +- src/registrar/tests/test_views_domain.py | 213 ++++++++++++++++++++--- src/registrar/views/domain.py | 6 +- 3 files changed, 193 insertions(+), 30 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 3dd2c72ac..5eb043742 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,4 +1,5 @@ """Forms for domain management.""" + import logging from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator @@ -264,8 +265,7 @@ class AuthorizingOfficialContactForm(ContactForm): if disable_maxlength: # Remove the maxlength dialog if "maxlength" in field.widget.attrs: - field.widget.attrs.pop('maxlength', None) - + field.widget.attrs.pop("maxlength", None) def save(self, commit=True): """ diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 8cc3b16a0..2a861a878 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1021,6 +1021,144 @@ class TestDomainAuthorizingOfficial(TestDomainOverview): self.assertEqual("Testy2", self.domain_information.authorizing_official.first_name) self.assertEqual(ao_pk, self.domain_information.authorizing_official.id) + def assert_all_form_fields_have_expected_values(self, form, test_cases, test_for_disabled=False): + """ + Asserts that each specified form field has the expected value and, optionally, checks if the field is disabled. + + This method iterates over a list of tuples, where each + tuple contains a field name and the expected value for that field. + It uses subtests to isolate each assertion, allowing multiple field + checks within a single test method without stopping at the first failure. + + Example usage: + test_cases = [ + ("first_name", "John"), + ("last_name", "Doe"), + ("email", "john.doe@example.com"), + ] + self.assert_all_form_fields_have_expected_values(my_form, test_cases, test_for_disabled=True) + """ + for field_name, expected_value in test_cases: + with self.subTest(field_name=field_name, expected_value=expected_value): + # Test that each field has the value we expect + self.assertEqual(expected_value, form[field_name].value) + + if test_for_disabled: + # Test for disabled on each field + self.assertTrue("disabled" in form[field_name].attrs) + + def test_domain_edit_authorizing_official_federal(self): + """Tests that no edit can occur when the underlying domain is federal""" + + # Set the org type to federal + self.domain_information.organization_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.save() + + # Add an AO. We can do this at the model level, just not the form level. + self.domain_information.authorizing_official = Contact( + first_name="Apple", last_name="Tester", title="CIO", email="nobody@igorville.gov" + ) + self.domain_information.authorizing_official.save() + self.domain_information.save() + + ao_page = self.app.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Test if the form is populating data correctly + ao_form = ao_page.forms[0] + + test_cases = [ + ("first_name", "Apple"), + ("last_name", "Tester"), + ("title", "CIO"), + ("email", "nobody@igorville.gov"), + ] + self.assert_all_form_fields_have_expected_values(ao_form, test_cases, test_for_disabled=True) + + # Attempt to change data on each field. Because this domain is federal, + # this should not succeed. + ao_form["first_name"] = "Orange" + ao_form["last_name"] = "Smoothie" + ao_form["title"] = "Cat" + ao_form["email"] = "somebody@igorville.gov" + + submission = ao_form.submit() + + # A 302 indicates this page underwent a redirect. + self.assertEqual(submission.status_code, 302) + + followed_submission = submission.follow() + + # Test the returned form for data accuracy. These values should be unchanged. + new_form = followed_submission.forms[0] + self.assert_all_form_fields_have_expected_values(new_form, test_cases, test_for_disabled=True) + + # refresh domain information. Test these values in the DB. + self.domain_information.refresh_from_db() + + # All values should be unchanged. These are defined manually for code clarity. + self.assertEqual("Apple", self.domain_information.authorizing_official.first_name) + self.assertEqual("Tester", self.domain_information.authorizing_official.last_name) + self.assertEqual("CIO", self.domain_information.authorizing_official.title) + self.assertEqual("nobody@igorville.gov", self.domain_information.authorizing_official.email) + + def test_domain_edit_authorizing_official_tribal(self): + """Tests that no edit can occur when the underlying domain is tribal""" + + # Set the org type to federal + self.domain_information.organization_type = DomainInformation.OrganizationChoices.TRIBAL + self.domain_information.save() + + # Add an AO. We can do this at the model level, just not the form level. + self.domain_information.authorizing_official = Contact( + first_name="Apple", last_name="Tester", title="CIO", email="nobody@igorville.gov" + ) + self.domain_information.authorizing_official.save() + self.domain_information.save() + + ao_page = self.app.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Test if the form is populating data correctly + ao_form = ao_page.forms[0] + + test_cases = [ + ("first_name", "Apple"), + ("last_name", "Tester"), + ("title", "CIO"), + ("email", "nobody@igorville.gov"), + ] + self.assert_all_form_fields_have_expected_values(ao_form, test_cases, test_for_disabled=True) + + # Attempt to change data on each field. Because this domain is federal, + # this should not succeed. + ao_form["first_name"] = "Orange" + ao_form["last_name"] = "Smoothie" + ao_form["title"] = "Cat" + ao_form["email"] = "somebody@igorville.gov" + + submission = ao_form.submit() + + # A 302 indicates this page underwent a redirect. + self.assertEqual(submission.status_code, 302) + + followed_submission = submission.follow() + + # Test the returned form for data accuracy. These values should be unchanged. + new_form = followed_submission.forms[0] + self.assert_all_form_fields_have_expected_values(new_form, test_cases, test_for_disabled=True) + + # refresh domain information. Test these values in the DB. + self.domain_information.refresh_from_db() + + # All values should be unchanged. These are defined manually for code clarity. + self.assertEqual("Apple", self.domain_information.authorizing_official.first_name) + self.assertEqual("Tester", self.domain_information.authorizing_official.last_name) + self.assertEqual("CIO", self.domain_information.authorizing_official.title) + self.assertEqual("nobody@igorville.gov", self.domain_information.authorizing_official.email) + def test_domain_edit_authorizing_official_creates_new(self): """When editing an authorizing official for domain information and AO IS joined to another object""" @@ -1087,7 +1225,7 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Not igorville") self.assertContains(success_result_page, "Faketown") - + def test_domain_org_name_address_form_tribal(self): """ Submitting a change to organization_name is blocked for tribal domains @@ -1113,11 +1251,11 @@ class TestDomainOrganization(TestDomainOverview): # Check the value of the input field organization_name_input = form.fields["organization_name"][0] self.assertEqual(organization_name_input.value, "Town of Igorville") - + # Check if the input field is disabled self.assertTrue("disabled" in organization_name_input.attrs) self.assertEqual(organization_name_input.attrs.get("disabled"), "") - + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] org_name_page.form["organization_name"] = "Not igorville" @@ -1133,12 +1271,22 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Town of Igorville") self.assertNotContains(success_result_page, "Not igorville") + # Do another check on the form itself + form = success_result_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["organization_name"][0] + self.assertEqual(organization_name_input.value, "Town of Igorville") + + # Check if the input field is disabled + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") + # Check for the value we want to update self.assertContains(success_result_page, "Faketown") - - def test_domain_org_name_address_form_federal(self): + + def test_domain_org_name_address_form_federal_disabled(self): """ - Submitting a change to federal_agency is blocked for federal domains + Tests if the federal_agency field is readonly """ # Set the current domain to a tribal organization with a preset value. # Save first, so we can test if saving is unaffected (it should be). @@ -1155,33 +1303,48 @@ class TestDomainOrganization(TestDomainOverview): self.assertEqual(self.domain_information.organization_type, federal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field - federal_agency_input = form.fields["federal_agency"][0] - self.assertEqual(federal_agency_input.value, "AMTRAK") - - # Check if the input field is disabled - self.assertTrue("disabled" in federal_agency_input.attrs) - self.assertEqual(federal_agency_input.attrs.get("disabled"), "") - - session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + organization_name_input = form.fields["federal_agency"][0] - org_name_page.form["organization_name"] = "Not igorville" - org_name_page.form["city"] = "Faketown" + # Check if the input field is disabled. + # Webtest has some issues dealing with Selects, so we can't + # directly test the value but we can test its attributes. + # This is done in another test. + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") - self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + def test_federal_agency_submit_blocked(self): + """ + Submitting a change to federal_agency is blocked for federal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + federal_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = federal_org_type + self.domain_information.save() - # Make the change. The org name should be unchanged, but city should be modifiable. - success_result_page = org_name_page.form.submit() - self.assertEqual(success_result_page.status_code, 200) + old_federal_agency_value = ("AMTRAK", "AMTRAK") + try: + # Add a federal agency. Defined as a tuple since this list may change order. + self.domain_information.federal_agency = old_federal_agency_value + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") - # Check for the old and new value - self.assertContains(success_result_page, "Town of Igorville") - self.assertNotContains(success_result_page, "Not igorville") + self.assertEqual(self.domain_information.organization_type, federal_org_type) - # Check for the value we want to update - self.assertContains(success_result_page, "Faketown") + new_value = ("Department of State", "Department of State") + self.client.post( + reverse("domain-org-name-address", kwargs={"pk": self.domain.id}), + { + "federal_agency": new_value, + }, + ) + self.assertEqual(self.domain_information.federal_agency, old_federal_agency_value) + self.assertNotEqual(self.domain_information.federal_agency, new_value) class TestDomainContactInformation(TestDomainOverview): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 07b695356..34628bc88 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -135,7 +135,7 @@ class DomainFormBaseView(DomainBaseView, FormMixin): # superclass has the redirect return super().form_invalid(form) - + def get_domain_info_from_domain(self) -> DomainInformation | None: """ Grabs the underlying domain_info object based off of self.object.name. @@ -147,7 +147,7 @@ class DomainFormBaseView(DomainBaseView, FormMixin): current_domain_info = _domain_info.get() else: logger.error("Could get domain_info. No domain info exists, or duplicates exist.") - + return current_domain_info @@ -237,7 +237,7 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): domain_info = self.get_domain_info_from_domain() invalid_fields = [DomainApplication.OrganizationChoices.FEDERAL, DomainApplication.OrganizationChoices.TRIBAL] is_federal_or_tribal = domain_info and (domain_info.organization_type in invalid_fields) - + form_kwargs["disable_fields"] = is_federal_or_tribal return form_kwargs From 707ceb286f6c97191451b28941cb3b0ec8d409a9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:53:19 -0700 Subject: [PATCH 39/92] Update _forms.scss --- src/registrar/assets/sass/_theme/_forms.scss | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_forms.scss b/src/registrar/assets/sass/_theme/_forms.scss index bfd738304..86ccd101f 100644 --- a/src/registrar/assets/sass/_theme/_forms.scss +++ b/src/registrar/assets/sass/_theme/_forms.scss @@ -39,8 +39,7 @@ legend.float-left-tablet + button.float-right-tablet { } } -/* Custom style for disabled inputs */ -// TODO - UPDATE THIS! +// Custom style for disabled inputs .usa-input:disabled, .usa-select:disabled, .usa-textarea:disabled { background-color: --body-fg; color: --close-button-hover-bg; From 06f5a7f322704192b8510fb4e328c66428b94153 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:58:13 -0700 Subject: [PATCH 40/92] Update domain_org_name_address.html --- src/registrar/templates/domain_org_name_address.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/registrar/templates/domain_org_name_address.html b/src/registrar/templates/domain_org_name_address.html index 587ba4782..3a5254346 100644 --- a/src/registrar/templates/domain_org_name_address.html +++ b/src/registrar/templates/domain_org_name_address.html @@ -11,6 +11,13 @@

          The name of your organization will be publicly listed as the domain registrant.

          + {% if domain.domain_info.organization_type == 'federal' %} +

          + The federal agency for your organization can’t be updated here. + To suggest an update, email help@get.gov. +

          + {% endif %} + {% include "includes/required_fields.html" %}
          From 6e25d0e000667972c77b6630a2234d1354877afa Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:58:13 -0700 Subject: [PATCH 41/92] Change federal agency to textinput --- src/registrar/forms/domain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 5eb043742..2d67ed427 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -351,11 +351,11 @@ class DomainOrgNameAddressForm(forms.ModelForm): }, } widgets = { - # We need to set the required attributed for federal_agency and - # state/territory because for these fields we are creating an individual + # We need to set the required attributed for State/territory + # because for this fields we are creating an individual # instance of the Select. For the other fields we use the for loop to set # the class's required attribute to true. - "federal_agency": forms.Select(attrs={"required": True}, choices=DomainInformation.AGENCY_CHOICES), + "federal_agency": forms.TextInput, "organization_name": forms.TextInput, "address_line1": forms.TextInput, "address_line2": forms.TextInput, From 8438067d79a4a864994a70d9ac12522df58d2bbb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 08:46:35 -0700 Subject: [PATCH 42/92] Update domain.py --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 2d67ed427..e01b2cd03 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -351,7 +351,7 @@ class DomainOrgNameAddressForm(forms.ModelForm): }, } widgets = { - # We need to set the required attributed for State/territory + # We need to set the required attributed for State/territory # because for this fields we are creating an individual # instance of the Select. For the other fields we use the for loop to set # the class's required attribute to true. From 4e67733861a778fb783858cd88d4caadd8656b68 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:44:13 -0700 Subject: [PATCH 43/92] Update test_views_domain.py --- src/registrar/tests/test_views_domain.py | 51 +++++++++++++++++------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 2a861a878..6da1ce66e 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1245,7 +1245,6 @@ class TestDomainOrganization(TestDomainOverview): self.assertEqual(self.domain_information.organization_type, tribal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) - print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field @@ -1284,38 +1283,62 @@ class TestDomainOrganization(TestDomainOverview): # Check for the value we want to update self.assertContains(success_result_page, "Faketown") - def test_domain_org_name_address_form_federal_disabled(self): + def test_domain_org_name_address_form_federal(self): """ - Tests if the federal_agency field is readonly + Submitting a change to federal_agency is blocked for federal domains """ # Set the current domain to a tribal organization with a preset value. # Save first, so we can test if saving is unaffected (it should be). - federal_org_type = DomainInformation.OrganizationChoices.FEDERAL - self.domain_information.organization_type = federal_org_type + fed_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = fed_org_type self.domain_information.save() try: - # Add a federal agency. Defined as a tuple since this list may change order. - self.domain_information.federal_agency = ("AMTRAK", "AMTRAK") + self.domain_information.federal_agency = "AMTRAK" self.domain_information.save() except ValueError as err: self.fail(f"A ValueError was caught during the test: {err}") - self.assertEqual(self.domain_information.organization_type, federal_org_type) + self.assertEqual(self.domain_information.organization_type, tribal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) - print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field - organization_name_input = form.fields["federal_agency"][0] + agency_input = form.fields["federal_agency"][0] + self.assertEqual(agency_input.value, "AMTRAK") - # Check if the input field is disabled. - # Webtest has some issues dealing with Selects, so we can't - # directly test the value but we can test its attributes. - # This is done in another test. + # Check if the input field is disabled + self.assertTrue("disabled" in agency_input.attrs) + self.assertEqual(agency_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["federal_agency"] = "Department of State" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The agency should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "AMTRAK") + self.assertNotContains(success_result_page, "Department of State") + + # Do another check on the form itself + form = success_result_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["federal_agency"][0] + self.assertEqual(organization_name_input.value, "AMTRAK") + + # Check if the input field is disabled self.assertTrue("disabled" in organization_name_input.attrs) self.assertEqual(organization_name_input.attrs.get("disabled"), "") + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") + def test_federal_agency_submit_blocked(self): """ Submitting a change to federal_agency is blocked for federal domains From e900007c957c8b874865599c750fc7aa38227a81 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:59:14 -0700 Subject: [PATCH 44/92] Update test_views_domain.py --- src/registrar/tests/test_views_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 6da1ce66e..6aede926f 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1298,7 +1298,7 @@ class TestDomainOrganization(TestDomainOverview): except ValueError as err: self.fail(f"A ValueError was caught during the test: {err}") - self.assertEqual(self.domain_information.organization_type, tribal_org_type) + self.assertEqual(self.domain_information.organization_type, fed_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) From 33b47d27d17a75d8f54281302b46f36bcab3c995 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:03:10 -0800 Subject: [PATCH 45/92] handle none --- src/registrar/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5b8d67983..68f27d15c 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,7 +376,10 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time = f"{avg_approval_time.days} days" + + avg_approval_time="No approvals to use" + if avg_approval_time is not None: + avg_approval_time = f"{avg_approval_time.days} days" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") From d1bac52aa61a1279788fc9430d593a4ec2968f1b Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:08:24 -0800 Subject: [PATCH 46/92] moved code line --- src/registrar/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 68f27d15c..4b841ab12 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -377,10 +377,11 @@ def analytics(request): ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time="No approvals to use" + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" - + else: + avg_approval_time="No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From bd352443964ebf6aae03a67c5ace122df07b03c7 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:33:22 -0800 Subject: [PATCH 47/92] lint --- src/registrar/admin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 4b841ab12..59aa2ace8 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,12 +376,11 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - - + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" else: - avg_approval_time="No approvals to use" + avg_approval_time = "No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From 454cac951ab6cf9f5cb51103092c3bb849f54408 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:41:40 -0700 Subject: [PATCH 48/92] Bug fix for PR --- src/registrar/assets/js/get-gov-admin.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 4ed1a0d28..8170e4bd0 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -61,18 +61,19 @@ function openInNewTab(el, removeAttribute = false){ * This intentionally does not interact with createPhantomModalFormButtons() */ (function (){ - function displayModalOnDropdownClick(linkClickedDisplaysModal, statusDropdown, cancelButton, valueToCheck){ + function displayModalOnDropdownClick(linkClickedDisplaysModal, statusDropdown, actionButton, valueToCheck){ // If these exist all at the same time, we're on the right page if (linkClickedDisplaysModal && statusDropdown && statusDropdown.value){ + + // Set the previous value in the event the user cancels. + let previousValue = statusDropdown.value; + if (actionButton){ - if (cancelButton){ - // Store the previous value in the event the user cancels. - // We only need to do this if cancel button is specified. - let previousValue = statusDropdown.value; - cancelButton.addEventListener('click', function() { + // Otherwise, if the confirmation buttion is pressed, set it to that + actionButton.addEventListener('click', function() { // Revert the dropdown to its previous value - statusDropdown.value = previousValue; + statusDropdown.value = valueToCheck; }); }else { console.log("displayModalOnDropdownClick() -> Cancel button was null") @@ -82,6 +83,10 @@ function openInNewTab(el, removeAttribute = false){ statusDropdown.addEventListener('change', function() { // Check if "Ineligible" is selected if (this.value && this.value.toLowerCase() === valueToCheck) { + // Set the old value in the event the user cancels, + // or otherwise exists the dropdown. + statusDropdown.value = previousValue + // Display the modal. linkClickedDisplaysModal.click() } @@ -98,9 +103,9 @@ function openInNewTab(el, removeAttribute = false){ // Because the modal button does not have the class "dja-form-placeholder", // it will not be affected by the createPhantomModalFormButtons() function. - let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); + let actionButton = document.querySelector('button[name="_set_application_ineligible"]'); let valueToCheck = "ineligible" - displayModalOnDropdownClick(modalButton, statusDropdown, cancelButton, valueToCheck); + displayModalOnDropdownClick(modalButton, statusDropdown, actionButton, valueToCheck); } hookModalToIneligibleStatus() From ee60be8b10845519ad88bc0f4147ca9eac8a5c27 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:21:08 -0600 Subject: [PATCH 49/92] Add semaphore + add error handling for .close --- src/epplibwrapper/client.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index a7856298b..823c26288 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -1,6 +1,7 @@ """Provide a wrapper around epplib to handle authentication and errors.""" import logging +from gevent.lock import BoundedSemaphore try: from epplib.client import Client @@ -52,6 +53,9 @@ class EPPLibWrapper: "urn:ietf:params:xml:ns:contact-1.0", ], ) + # We should only ever have one active connection at a time, + # given that + self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() except Exception: @@ -91,12 +95,23 @@ class EPPLibWrapper: raise RegistryError(message) from err def _disconnect(self) -> None: - """Close the connection.""" + """Close the connection. Sends a logout command and closes the connection.""" + self._send_logout_command() + self._close_client() + + def _send_logout_command(self): + """Sends a logout command to epp""" try: self._client.send(commands.Logout()) # type: ignore - self._client.close() # type: ignore - except Exception: - logger.warning("Connection to registry was not cleanly closed.") + except Exception as err: + logger.warning(f"Logout command not sent successfully: {err}") + + def _close_client(self): + """Closes an active client connection""" + try: + self._client.close() + except Exception as err: + logger.warning(f"Connection to registry was not cleanly closed: {err}") def _send(self, command): """Helper function used by `send`.""" @@ -146,6 +161,8 @@ class EPPLibWrapper: cmd_type = command.__class__.__name__ if not cleaned: raise ValueError("Please sanitize user input before sending it.") + + self.connection_lock.acquire() try: return self._send(command) except RegistryError as err: @@ -161,6 +178,8 @@ class EPPLibWrapper: return self._retry(command) else: raise err + finally: + self.connection_lock.release() try: From 3b9f68ac4b58bb1f7b04f797d9eb2b8ddd3b6b4c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:59:55 -0600 Subject: [PATCH 50/92] linting --- src/epplibwrapper/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 823c26288..5006708d6 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -54,7 +54,7 @@ class EPPLibWrapper: ], ) # We should only ever have one active connection at a time, - # given that + # given that self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() @@ -105,7 +105,7 @@ class EPPLibWrapper: self._client.send(commands.Logout()) # type: ignore except Exception as err: logger.warning(f"Logout command not sent successfully: {err}") - + def _close_client(self): """Closes an active client connection""" try: @@ -161,7 +161,7 @@ class EPPLibWrapper: cmd_type = command.__class__.__name__ if not cleaned: raise ValueError("Please sanitize user input before sending it.") - + self.connection_lock.acquire() try: return self._send(command) From 6992f1489b30c087118576b61dadd9eff0e88b1d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:01:01 -0600 Subject: [PATCH 51/92] Fix comment --- src/epplibwrapper/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 5006708d6..b346563d2 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -53,8 +53,7 @@ class EPPLibWrapper: "urn:ietf:params:xml:ns:contact-1.0", ], ) - # We should only ever have one active connection at a time, - # given that + # We should only ever have one active connection at a time self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() From f31d6ee0735e1014d3dd7c7e553c213ff48fac7d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:44:56 -0600 Subject: [PATCH 52/92] Update test_admin.py --- src/registrar/tests/test_admin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index fb7e7af5e..8d983dcb7 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -1224,7 +1224,7 @@ class TestDomainRequestAdmin(MockEppLib): EMAIL = "mayor@igorville.gov" User.objects.filter(email=EMAIL).delete() - # Create a sample application + # Create a sample domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW) p = "userpass" @@ -1261,7 +1261,7 @@ class TestDomainRequestAdmin(MockEppLib): # Test that approved domain exists and equals requested domain self.assertEqual(domain_request.creator.status, "restricted") - # 'Get' to the application again + # 'Get' to the domain request again response = self.client.get( "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True, @@ -1282,7 +1282,7 @@ class TestDomainRequestAdmin(MockEppLib): EMAIL = "mayor@igorville.gov" User.objects.filter(email=EMAIL).delete() - # Create a sample application + # Create a sample domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW) p = "userpass" @@ -1310,7 +1310,7 @@ class TestDomainRequestAdmin(MockEppLib): "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): - # Modify the application's property + # Modify the domain request's property domain_request.status = DomainRequest.DomainRequestStatus.INELIGIBLE # Use the model admin's save_model method @@ -1319,7 +1319,7 @@ class TestDomainRequestAdmin(MockEppLib): # Test that approved domain exists and equals requested domain self.assertEqual(domain_request.creator.status, "restricted") - # 'Get' to the application again + # 'Get' to the domain request again response = self.client.get( "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True, From 70bfd4343e9f42eac5761686c6321b99547f8a8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:25:46 -0600 Subject: [PATCH 53/92] Add test, more semaphore(s) --- src/epplibwrapper/client.py | 21 +++-- src/epplibwrapper/tests/test_client.py | 105 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index b346563d2..5ca2b5c26 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -55,15 +55,20 @@ class EPPLibWrapper: ) # We should only ever have one active connection at a time self.connection_lock = BoundedSemaphore(1) + + self.connection_lock.acquire() try: self._initialize_client() except Exception: logger.warning("Unable to configure epplib. Registrar cannot contact registry.") + finally: + self.connection_lock.release() def _initialize_client(self) -> None: """Initialize a client, assuming _login defined. Sets _client to initialized client. Raises errors if initialization fails. This method will be called at app initialization, and also during retries.""" + # establish a client object with a TCP socket transport # note that type: ignore added in several places because linter complains # about _client initially being set to None, and None type doesn't match code @@ -77,11 +82,7 @@ class EPPLibWrapper: ) try: # use the _client object to connect - self._client.connect() # type: ignore - response = self._client.send(self._login) # type: ignore - if response.code >= 2000: # type: ignore - self._client.close() # type: ignore - raise LoginError(response.msg) # type: ignore + self._connect() except TransportError as err: message = "_initialize_client failed to execute due to a connection error." logger.error(f"{message} Error: {err}") @@ -93,6 +94,15 @@ class EPPLibWrapper: logger.error(f"{message} Error: {err}") raise RegistryError(message) from err + def _connect(self) -> None: + """Connects to EPP. Sends a login command. If an invalid response is returned, + the client will be closed and a LoginError raised.""" + self._client.connect() # type: ignore + response = self._client.send(self._login) # type: ignore + if response.code >= 2000: # type: ignore + self._client.close() # type: ignore + raise LoginError(response.msg) # type: ignore + def _disconnect(self) -> None: """Close the connection. Sends a logout command and closes the connection.""" self._send_logout_command() @@ -115,7 +125,6 @@ class EPPLibWrapper: def _send(self, command): """Helper function used by `send`.""" cmd_type = command.__class__.__name__ - try: # check for the condition that the _client was not initialized properly # at app initialization diff --git a/src/epplibwrapper/tests/test_client.py b/src/epplibwrapper/tests/test_client.py index f95b37dcd..17ae6c2cf 100644 --- a/src/epplibwrapper/tests/test_client.py +++ b/src/epplibwrapper/tests/test_client.py @@ -1,4 +1,7 @@ +import datetime +from dateutil.tz import tzlocal # type: ignore from unittest.mock import MagicMock, patch +from pathlib import Path from django.test import TestCase from epplibwrapper.client import EPPLibWrapper from epplibwrapper.errors import RegistryError, LoginError @@ -8,6 +11,10 @@ import logging try: from epplib.exceptions import TransportError from epplib.responses import Result + from epplib.client import Client + from epplib.transport import SocketTransport + from epplib import commands + from epplib.models import common, info except ImportError: pass @@ -255,3 +262,101 @@ class TestClient(TestCase): mock_close.assert_called_once() # send() is called 5 times: send(login), send(command) fail, send(logout), send(login), send(command) self.assertEquals(mock_send.call_count, 5) + + def test_send_command_close_failure_recovers(self): + """Test when the .close on a connection fails and a .send follows suit. + Flow: + Initialization succeeds + Send command fails (with 2400 code) prompting retry + Client closes and re-initializes, and command succeeds""" + + expected_result = { + "cl_tr_id": None, + "code": 1000, + "extensions": [], + "msg": "Command completed successfully", + "msg_q": None, + "res_data": [ + info.InfoDomainResultData( + roid="DF1340360-GOV", + statuses=[ + common.Status( + state="serverTransferProhibited", + description=None, + lang="en", + ), + common.Status(state="inactive", description=None, lang="en"), + ], + cl_id="gov2023-ote", + cr_id="gov2023-ote", + cr_date=datetime.datetime(2023, 8, 15, 23, 56, 36, tzinfo=tzlocal()), + up_id="gov2023-ote", + up_date=datetime.datetime(2023, 8, 17, 2, 3, 19, tzinfo=tzlocal()), + tr_date=None, + name="test3.gov", + registrant="TuaWnx9hnm84GCSU", + admins=[], + nsset=None, + keyset=None, + ex_date=datetime.date(2024, 8, 15), + auth_info=info.DomainAuthInfo(pw="2fooBAR123fooBaz"), + ) + ], + "sv_tr_id": "wRRNVhKhQW2m6wsUHbo/lA==-29a", + } + + def fake_receive(command, cleaned=None): + location = Path(__file__).parent / "utility" / "infoDomain.xml" + xml = (location).read_bytes() + return xml + + def fake_success_send(self, command, cleaned=None): + mock = MagicMock( + code=1000, + msg="Command completed successfully", + res_data=None, + cl_tr_id="xkw1uo#2023-10-17T15:29:09.559376", + sv_tr_id="5CcH4gxISuGkq8eqvr1UyQ==-35a", + extensions=[], + msg_q=None, + ) + return mock + + def fake_failure_send(self, command, cleaned=None): + mock = MagicMock( + code=2400, + msg="Command failed", + res_data=None, + cl_tr_id="xkw1uo#2023-10-17T15:29:09.559376", + sv_tr_id="5CcH4gxISuGkq8eqvr1UyQ==-35a", + extensions=[], + msg_q=None, + ) + return mock + + def do_nothing(command): + pass + + wrapper = None + # Trigger a retry + # Do nothing on connect, as we aren't testing it and want to connect while + # mimicking the rest of the client as closely as possible (which is not entirely possible with MagicMock) + with patch.object(EPPLibWrapper, "_connect", do_nothing): + with patch.object(SocketTransport, "send", fake_failure_send): + wrapper = EPPLibWrapper() + tested_command = commands.InfoDomain(name="test.gov") + try: + wrapper.send(tested_command, cleaned=True) + wrapper._retry(tested_command) + except RegistryError as err: + expected_error = "InfoDomain failed to execute due to a connection error." + self.assertEqual(err.args[0], expected_error) + else: + self.fail("Registry error was not thrown") + + with patch.object(EPPLibWrapper, "_connect", do_nothing): + with patch.object(SocketTransport, "send", fake_success_send), patch.object( + SocketTransport, "receive", fake_receive + ): + result = wrapper.send(tested_command, cleaned=True) + self.assertEqual(expected_result, result.__dict__) From 5cfd6193fd84e5ea39d781666f838e5cbe2cc277 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:28:21 -0600 Subject: [PATCH 54/92] Update client.py --- src/epplibwrapper/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 5ca2b5c26..a130d8bfc 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -68,7 +68,6 @@ class EPPLibWrapper: """Initialize a client, assuming _login defined. Sets _client to initialized client. Raises errors if initialization fails. This method will be called at app initialization, and also during retries.""" - # establish a client object with a TCP socket transport # note that type: ignore added in several places because linter complains # about _client initially being set to None, and None type doesn't match code @@ -125,6 +124,7 @@ class EPPLibWrapper: def _send(self, command): """Helper function used by `send`.""" cmd_type = command.__class__.__name__ + try: # check for the condition that the _client was not initialized properly # at app initialization From 763a1c51fc52367e11d371a16793c535c9c45198 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:53:30 -0600 Subject: [PATCH 55/92] Merge conflict fixes --- src/registrar/assets/js/get-gov-admin.js | 4 +- .../admin/domain_application_change_form.html | 8 +-- src/registrar/tests/test_admin.py | 58 ------------------- 3 files changed, 6 insertions(+), 64 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 8170e4bd0..4ed00c33f 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -57,7 +57,7 @@ function openInNewTab(el, removeAttribute = false){ createPhantomModalFormButtons(); })(); -/** An IIFE for DomainApplication to hook a modal to a dropdown option. +/** An IIFE for DomainRequest to hook a modal to a dropdown option. * This intentionally does not interact with createPhantomModalFormButtons() */ (function (){ @@ -103,7 +103,7 @@ function openInNewTab(el, removeAttribute = false){ // Because the modal button does not have the class "dja-form-placeholder", // it will not be affected by the createPhantomModalFormButtons() function. - let actionButton = document.querySelector('button[name="_set_application_ineligible"]'); + let actionButton = document.querySelector('button[name="_set_domain_request_ineligible"]'); let valueToCheck = "ineligible" displayModalOnDropdownClick(modalButton, statusDropdown, actionButton, valueToCheck); } diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index f0e4cfe4f..95392da1e 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -27,7 +27,7 @@ class="usa-modal" id="toggle-set-ineligible" aria-labelledby="Are you sure you want to select ineligible status?" - aria-describedby="This application will be marked as ineligible." + aria-describedby="This request will be marked as ineligible." >
          @@ -51,7 +51,7 @@ Domain: {{ original.requested_domain.name }} {# Acts as a
          #}
          - New status: {{ original.ApplicationStatus.INELIGIBLE|capfirst }} + New status: {{ original.DomainRequestStatus.INELIGIBLE|capfirst }}

          @@ -61,7 +61,7 @@
        @@ -87,7 +87,7 @@
      • @@ -132,7 +132,7 @@ data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}" data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}" > -

        Chart: Unanaged domains

        +

        Chart: Unmanaged domains

        {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

        • diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index dd7e27f33..49fb59e79 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -64,12 +64,11 @@
        {% endfor %} +
        +

        Analytics

        + Dashboard +
        {% else %}

        {% translate 'You don’t have permission to view or edit anything.' %}

        {% endif %} - -
        -

        Analytics

        - Dashboard -
        diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index f1c7841d1..48b42f47c 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -475,7 +475,7 @@ class AuditedAdminMockData: class MockDb(TestCase): - """Hardcoded mocks make test case assertions sraightforward.""" + """Hardcoded mocks make test case assertions straightforward.""" def setUp(self): super().setUp() @@ -622,19 +622,19 @@ class MockDb(TestCase): ) with less_console_noise(): - self.domain_request_1 = completed_application( + self.domain_request_1 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" ) - self.domain_request_2 = completed_application( + self.domain_request_2 = completed_domain_request( status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" ) - self.domain_request_3 = completed_application( + self.domain_request_3 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" ) - self.domain_request_4 = completed_application( + self.domain_request_4 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" ) - self.domain_request_5 = completed_application( + self.domain_request_5 = completed_domain_request( status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" ) self.domain_request_3.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 475076711..b91f3bd18 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,6 +5,8 @@ from io import StringIO from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain from registrar.utility.csv_export import ( + export_data_managed_domains_to_csv, + export_data_unmanaged_domains_to_csv, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -530,68 +532,10 @@ class ExportDataTest(MockDb, MockEppLib): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.start_date, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - # Call the export functions - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.end_date, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_managed_domains_end_date, - get_domain_managers=True, - should_write_header=True, + export_data_managed_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -627,68 +571,10 @@ class ExportDataTest(MockDb, MockEppLib): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.start_date, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) - # Call the export functions - writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.end_date, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_end_date, - get_domain_managers=False, - should_write_header=True, + export_data_unmanaged_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -696,12 +582,12 @@ class ExportDataTest(MockDb, MockEppLib): self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( - "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "UNMANAGED DOMAINS AT START DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" - "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "UNMANAGED DOMAINS AT END DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" @@ -729,16 +615,17 @@ class ExportDataTest(MockDb, MockEppLib): csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition + # We'll skip submission date because it's dynamic and therefore + # impossible to set in expected_content columns = [ "Requested domain", "Organization type", - "Submission date", ] sort_fields = [ "requested_domain__name", ] filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, "submission_date__gte": self.start_date, } @@ -750,9 +637,9 @@ class ExportDataTest(MockDb, MockEppLib): # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( - "Requested domain,Organization type,Submission date\n" - "city3.gov,Federal - Executive,2024-03-05\n" - "city4.gov,Federal - Executive,2024-03-05\n" + "Requested domain,Organization type\n" + "city3.gov,Federal - Executive\n" + "city4.gov,Federal - Executive\n" ) # Normalize line endings and remove commas, @@ -785,16 +672,22 @@ class HelperFunctions(MockDb): "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + # Test with distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True) expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + # Test without distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1] + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" with less_console_noise(): filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 060c39804..e8746eafb 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from collections import Counter import csv import logging from datetime import datetime @@ -25,7 +26,8 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") + DomainInformation.objects.select_related("domain", "authorizing_official") + .prefetch_related("domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) .distinct() @@ -190,7 +192,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() + requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -236,10 +238,10 @@ def write_requests_csv( """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. Works with write_header as long as the same writer object is passed.""" - all_requetsts = get_requests(filter_condition, sort_fields) + all_requests = get_requests(filter_condition, sort_fields) # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_requetsts, 1000) + paginator = Paginator(all_requests, 1000) for page_num in paginator.page_range: page = paginator.page(page_num) @@ -443,26 +445,37 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): ) -def get_sliced_domains(filter_condition): - """Get fitered domains counts sliced by org type and election office.""" +def get_sliced_domains(filter_condition, distinct=False): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count() - state_or_territory = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = domains.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct domain names based on filter condition + domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ domains_count, @@ -478,26 +491,34 @@ def get_sliced_domains(filter_condition): ] -def get_sliced_requests(filter_condition): - """Get fitered requests counts sliced by org type and election office.""" +def get_sliced_requests(filter_condition, distinct=False): + """Get filtered requests counts sliced by org type and election office.""" - requests = DomainRequest.objects.all().filter(**filter_condition).distinct() - requests_count = requests.count() - federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = requests.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct requests based on filter condition + requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ requests_count, @@ -531,7 +552,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( @@ -555,7 +576,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( @@ -604,7 +625,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True) writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) writer.writerow( @@ -628,7 +649,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True) writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) writer.writerow( diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04fcaa6f2..eba8423ed 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -2,6 +2,12 @@ from django.http import HttpResponse from django.views import View +from django.shortcuts import render +from django.contrib import admin +from django.db.models import Avg, F +from .. import models +import datetime +from django.utils import timezone from registrar.utility import csv_export @@ -10,6 +16,129 @@ import logging logger = logging.getLogger(__name__) +class AnalyticsView(View): + def get(self, request): + thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) + thirty_days_ago = timezone.make_aware(thirty_days_ago) + + last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) + last_30_days_approved_applications = models.DomainRequest.objects.filter( + created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED + ) + avg_approval_time = last_30_days_approved_applications.annotate( + approval_time=F("approved_domain__created_at") - F("submission_date") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # Format the timedelta to display only days + if avg_approval_time is not None: + avg_approval_time_display = f"{avg_approval_time.days} days" + else: + avg_approval_time_display = "No approvals to use" + + # The start and end dates are passed as url params + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains( + filter_unmanaged_domains_start_date, True + ) + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True) + + filter_ready_domains_start_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + filter_ready_domains_end_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": start_date_formatted, + } + filter_deleted_domains_end_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + filter_requests_start_date = { + "created_at__lte": start_date_formatted, + } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_start_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + filter_submitted_requests_end_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + + context = dict( + # Generate a dictionary of context variables that are common across all admin templates + # (site_header, site_url, ...), + # include it in the larger context dictionary so it's available in the template rendering context. + # This ensures that the admin interface styling and behavior are consistent with other admin pages. + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(), + last_30_days_applications=last_30_days_applications.count(), + last_30_days_approved_applications=last_30_days_approved_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time_display, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, + ), + ) + return render(request, "admin/analytics.html", context) + + class ExportDataType(View): def get(self, request, *args, **kwargs): # match the CSV example with all the fields From 9ecd34593c15b439e9b9d09312cd798e6a73969d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Mar 2024 18:31:58 -0400 Subject: [PATCH 67/92] make update charts button stand out more --- src/registrar/templates/admin/analytics.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2c5963e75..e73f22ec5 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -105,7 +105,7 @@
      • -