From 5723cb1fea490d885778fba7cff220d2bf0cfad8 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Tue, 12 Dec 2023 12:30:39 -0600 Subject: [PATCH 001/125] WIP analytics page in Admin --- src/registrar/admin.py | 59 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 12 ++++ 2 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 src/registrar/templates/admin/analytics.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c5f5be276..2156aeeb1 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,15 +1,17 @@ import logging +import datetime + from django import forms -from django.db.models.functions import Concat +from django.db.models.functions import Concat, Avg, F from django.http import HttpResponse -from django.shortcuts import redirect +from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType -from django.http.response import HttpResponseRedirect -from django.urls import reverse +from django.http.response import HttpResponse, HttpResponseRedirect +from django.urls import path, reverse from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.domain import Domain from registrar.models.user import User @@ -353,6 +355,39 @@ class MyUserAdmin(BaseUserAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] + def get_urls(self): + urlpatterns = super().get_urls() + + my_urls = [ + path( + "analytics/", + self.admin_site.admin_view(self.user_analytics), + name="user_analytics", + ), + ] + + return my_urls + urlpatterns + + def user_analytics(self, request): + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + context = dict( + **self.admin_site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + ), + ) + return render(request, "admin/analytics.html", context) + # Let's define First group # (which should in theory be the ONLY group) def group(self, obj): @@ -1095,8 +1130,6 @@ class DomainAdmin(ListHeaderAdmin): return response def get_urls(self): - from django.urls import path - urlpatterns = super().get_urls() # Used to extrapolate a path name, for instance @@ -1178,9 +1211,11 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - "Error deleting this Domain: " - f"Can't switch from state '{obj.state}' to 'deleted'" - ", must be either 'dns_needed' or 'on_hold'", + ( + "Error deleting this Domain: " + f"Can't switch from state '{obj.state}' to 'deleted'" + ", must be either 'dns_needed' or 'on_hold'" + ), messages.ERROR, ) except Exception: @@ -1192,7 +1227,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("Domain %s has been deleted. Thanks!") % obj.name, + "Domain %s has been deleted. Thanks!" % obj.name, ) return HttpResponseRedirect(".") @@ -1234,7 +1269,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("%s is in client hold. This domain is no longer accessible on the public internet.") % obj.name, + "%s is in client hold. This domain is no longer accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") @@ -1263,7 +1298,7 @@ class DomainAdmin(ListHeaderAdmin): else: self.message_user( request, - ("%s is ready. This domain is accessible on the public internet.") % obj.name, + "%s is ready. This domain is accessible on the public internet." % obj.name, ) return HttpResponseRedirect(".") diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html new file mode 100644 index 000000000..2906af223 --- /dev/null +++ b/src/registrar/templates/admin/analytics.html @@ -0,0 +1,12 @@ +{% extends "admin/base_site.html" %} + +{% block content_title %}Registrar Analytics{% endblock %} + +{% block content %} + +{% endblock %} From 348d77726035f73e4cd817429e741b4439aa39f0 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Mon, 5 Feb 2024 13:22:17 -0500 Subject: [PATCH 002/125] fix django function imports --- src/registrar/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 2156aeeb1..aa9795eac 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2,7 +2,8 @@ import logging import datetime from django import forms -from django.db.models.functions import Concat, Avg, F +from django.db.models import Avg, F +from django.db.models.functions import Concat from django.http import HttpResponse from django.shortcuts import redirect, render from django_fsm import get_available_FIELD_transitions From fb1735e23fec05bd016e833aca427dff4a2d2fff Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 22 Feb 2024 16:51:36 -0800 Subject: [PATCH 003/125] Update yaml to include metadata, readme with new directions an d pseudocode for s3 and SES work --- .github/workflows/daily-csv-upload.yaml | 9 +++ docs/developer/README.md | 9 +-- .../generate_current_metadata_report.py | 67 +++++++++++++++++++ src/registrar/utility/email.py | 4 ++ 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 src/registrar/management/commands/generate_current_metadata_report.py diff --git a/.github/workflows/daily-csv-upload.yaml b/.github/workflows/daily-csv-upload.yaml index 724a19457..84881398e 100644 --- a/.github/workflows/daily-csv-upload.yaml +++ b/.github/workflows/daily-csv-upload.yaml @@ -31,3 +31,12 @@ jobs: cf_space: ${{ secrets.CF_REPORT_ENV }} cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full" + - name: Generate current-metadata.csv + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets[env.CF_USERNAME] }} + cf_password: ${{ secrets[env.CF_PASSWORD] }} + cf_org: cisa-dotgov + cf_space: ${{ secrets.CF_REPORT_ENV }} + cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_metadata_report' --name metadata" + diff --git a/docs/developer/README.md b/docs/developer/README.md index dc4c9ddd2..7dc64ae56 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -330,11 +330,12 @@ To associate a S3 instance to your sandbox, follow these steps: 3. Click `Services` on the application nav bar 4. Add a new service (plus symbol) 5. Click `Marketplace Service` -6. On the `Select the service` dropdown, select `s3` -7. Under the dropdown on `Select Plan`, select `basic-sandbox` -8. Under `Service Instance` enter `getgov-s3` for the name +6. For Space, put in your sandbox initials +7. On the `Select the service` dropdown, select `s3` +8. Under the dropdown on `Select Plan`, select `basic-sandbox` +9. Under `Service Instance` enter `getgov-s3` for the name and leave the other fields empty -See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. +See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. The basic commands should be `cf bind-service getgov- ` and `cf restage getgov-`. ### Testing your S3 instance locally To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows: diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py new file mode 100644 index 000000000..d8f5a4693 --- /dev/null +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -0,0 +1,67 @@ +"""Generates current-metadata.csv then uploads to S3 + sends email""" + +import logging +import os + +from django.core.management import BaseCommand +from registrar.utility import csv_export +from registrar.utility.s3_bucket import S3ClientHelper + + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = ( + "Generates and uploads a current-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." + ) + + def add_arguments(self, parser): + """Add our two filename arguments.""" + parser.add_argument("--directory", default="migrationdata", help="Desired directory") + parser.add_argument( + "--checkpath", + default=True, + help="Flag that determines if we do a check for os.path.exists. Used for test cases", + ) + + def handle(self, **options): + """Grabs the directory then creates current-metadata.csv in that directory""" + file_name = "current-metadata.csv" + # Ensures a slash is added + directory = os.path.join(options.get("directory"), "") + check_path = options.get("checkpath") + + logger.info("Generating report...") + try: + self.generate_current_metadata_report(directory, file_name, check_path) + except Exception as err: + # TODO - #1317: Notify operations when auto report generation fails + raise err + else: + logger.info(f"Success! Created {file_name}") + + def generate_current_metadata_report(self, directory, file_name, check_path): + """Creates a current-full.csv file under the specified directory, + then uploads it to a AWS S3 bucket""" + s3_client = S3ClientHelper() + file_path = os.path.join(directory, file_name) + + # Generate a file locally for upload + with open(file_path, "w") as file: + csv_export.export_data_type_to_csv(file) + + if check_path and not os.path.exists(file_path): + raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") + + # Upload this generated file for our S3 instance + s3_client.upload_file(file_path, file_name) + """ + We want to make sure to upload to s3 for back up + And now we also want to get the file and encrypt it so we can send it in an email + """ + # metadata_file = s3_client.get_file(file_name) + # metadata_file.encryptthisthingherewithpyzipper + # email.blasend_email(metadata_file) + + + diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 461637f23..e4e997d9d 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -41,6 +41,7 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr raise EmailSendingError("Could not access the SES client.") from exc try: + #if not attachment: ses_client.send_email( FromEmailAddress=settings.DEFAULT_FROM_EMAIL, Destination={"ToAddresses": [to_address]}, @@ -51,5 +52,8 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr }, }, ) + # else: # has attachment + # same as above but figure out how to attach a file + # via boto3 "boto3 SES file attachment" except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc From 010853a5f038047d0ee5a0b1b09b4848e101f233 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 23 Feb 2024 15:26:53 -0800 Subject: [PATCH 004/125] Update readme directions for s3 --- docs/developer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/README.md b/docs/developer/README.md index 7dc64ae56..53e1182c4 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -335,7 +335,7 @@ To associate a S3 instance to your sandbox, follow these steps: 8. Under the dropdown on `Select Plan`, select `basic-sandbox` 9. Under `Service Instance` enter `getgov-s3` for the name and leave the other fields empty -See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. The basic commands should be `cf bind-service getgov- ` and `cf restage getgov-`. +See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. ### Testing your S3 instance locally To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows: From 56cd0b6d156f0270f78daf8150e8d79fee1a1b33 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 27 Feb 2024 18:43:59 -0500 Subject: [PATCH 005/125] Gather all existing reports on analytics page --- src/registrar/admin.py | 90 +++++++++---------- src/registrar/assets/sass/_theme/_admin.scss | 3 +- src/registrar/templates/admin/analytics.html | 63 +++++++++++-- src/registrar/templates/admin/index.html | 33 ------- .../django/admin/domain_change_list.html | 23 ----- 5 files changed, 99 insertions(+), 113 deletions(-) delete mode 100644 src/registrar/templates/admin/index.html delete mode 100644 src/registrar/templates/django/admin/domain_change_list.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index aa9795eac..94b8e3dfd 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -357,17 +357,58 @@ class MyUserAdmin(BaseUserAdmin): ordering = ["first_name", "last_name", "email"] def get_urls(self): + """Map a new page in admin for analytics.""" urlpatterns = super().get_urls() + # Used to extrapolate a path name, for instance + # name="{app_label}_{model_name}_export_data_type" + domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name + my_urls = [ path( "analytics/", self.admin_site.admin_view(self.user_analytics), name="user_analytics", ), + path( + "export_data_type/", + self.export_data_type, + name="%s_%s_export_data_type" % domain_path_meta, + ), + path( + "export_data_full/", + self.export_data_full, + name="%s_%s_export_data_full" % domain_path_meta, + ), + path( + "export_data_federal/", + self.export_data_federal, + name="%s_%s_export_data_federal" % domain_path_meta, + ), ] return my_urls + urlpatterns + + def export_data_type(self, request): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + + def export_data_full(self, request): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + + def export_data_federal(self, request): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response def user_analytics(self, request): last_30_days_applications = models.DomainApplication.objects.filter( @@ -1103,60 +1144,11 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def get_urls(self): - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - info = self.model._meta.app_label, self.model._meta.model_name - - my_url = [ - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % info, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % info, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % info, - ), - ] - - return my_url + urlpatterns - def response_change(self, request, obj): # Create dictionary of action functions ACTION_FUNCTIONS = { diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 760c4f13a..04dceef08 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -112,7 +112,8 @@ html[data-theme="light"] { .change-list .usa-table thead th, body.dashboard, body.change-list, - body.change-form { + body.change-form, + .analytics { color: var(--body-fg); } } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2906af223..82081d629 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,12 +1,61 @@ {% extends "admin/base_site.html" %} -{% block content_title %}Registrar Analytics{% endblock %} + + +{% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} -
    -
  • User Count: {{ data.user_count }}
  • -
  • Domain Count: {{ data.domain_count }}
  • -
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
  • -
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
  • -
+ + {% block object-tools %} + + {% endblock %} + +
+
+

At a glance

+
+
    +
  • User Count: {{ data.user_count }}
  • +
  • Domain Count: {{ data.domain_count }}
  • +
  • Domain applications (last 30 days): {{ data.applications_last_30_days }}
  • +
  • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
  • +
+
+
+ +
+

Domain growth

+
+ {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + The challenge is in the path definition in urls. It does NOT like admin/export_data/ + + See the commit "Review for ticket #999" + {% endcomment %} +
+
+ + +
+
+ + +
+ +
+
+
+ +
{% endblock %} diff --git a/src/registrar/templates/admin/index.html b/src/registrar/templates/admin/index.html deleted file mode 100644 index 04601ef32..000000000 --- a/src/registrar/templates/admin/index.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends "admin/index.html" %} - -{% block content %} -
- {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} -
-

Reports

-

Domain growth report

- - {% comment %} - Inputs of type date suck for accessibility. - We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. Itdoes NOT like admin/export_data/ - - See the commit "Review for ticket #999" - {% endcomment %} - -
-
- - -
-
- - -
- - -
- -
-
-{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html deleted file mode 100644 index 22df74685..000000000 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "admin/change_list.html" %} - -{% block object-tools %} - - -{% endblock %} \ No newline at end of file From fdb7fb659464a199972e4948e6dd544088e6ac1a Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 28 Feb 2024 09:17:59 -0800 Subject: [PATCH 006/125] Biz logic --- .../generate_current_metadata_report.py | 28 ++++++++-- src/registrar/utility/email.py | 54 +++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index d8f5a4693..69d111f4c 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -2,10 +2,12 @@ import logging import os +import pyzipper from django.core.management import BaseCommand from registrar.utility import csv_export from registrar.utility.s3_bucket import S3ClientHelper +from ...utility.email import send_templated_email, EmailSendingError logger = logging.getLogger(__name__) @@ -59,9 +61,27 @@ class Command(BaseCommand): We want to make sure to upload to s3 for back up And now we also want to get the file and encrypt it so we can send it in an email """ - # metadata_file = s3_client.get_file(file_name) - # metadata_file.encryptthisthingherewithpyzipper - # email.blasend_email(metadata_file) + unencrypted_metadata_input = s3_client.get_file(file_name) + + # Encrypt metadata into a zip file + + # pre-setting zip file name + encrypted_metadata_output = 'encrypted_metadata.zip' + # set this to be an env var somewhere + password = b'somepasswordhere' + # encrypted_metadata is the encrypted output + encrypted_metadata = _encrypt_metadata(unencrypted_metadata_input, encrypted_metadata_output, password) + print("encrypted_metadata is:", encrypted_metadata) + + # Send the metadata file that is zipped + # Q: Would we set the vars I set in email.py here to pass in to the helper function or best way to invoke + # send_templated_email(encrypted_metadata, attachment=True) - + def _encrypt_metadata(input_file, output_file, password): + with open(input_file, 'rb') as f_in: + with pyzipper.AESZipFile(output_file, 'w', compression=pyzipper.ZIP_LZMA, encryption=pyzipper.WZ_AES) as f_out: + f_out.setpassword(password) + f_out.writestr(input_file, f_in.read()) + return output_file + diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index e4e997d9d..199a6c304 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -40,6 +40,8 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr except Exception as exc: raise EmailSendingError("Could not access the SES client.") from exc + # Are we okay with passing in "attachment" var in as boolean parameter + # If so, TODO: add attachment boolean to other functions try: #if not attachment: ses_client.send_email( @@ -55,5 +57,57 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr # else: # has attachment # same as above but figure out how to attach a file # via boto3 "boto3 SES file attachment" + # we also want this to only send to the help email + + # from email.mime.multipart import MIMEMultipart + # from email.mime.text import MIMEText + # from email.mime.application import MIMEApplication + + # sender_email = 'sender@example.com' + # recipient_email = 'help@get.gov' + # subject = 'DOTGOV-Full Domain Metadata' + # body = 'Domain metadata email, should have an attachment included change here later.' + # attachment_path = 'path/to/attachment/file.pdf' + # aws_region = 'sesv2' + + # response = send_email_with_attachment(sender_email, recipient_email, subject, body, attachment_path, aws_region) + # print(response) except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc + + +# def send_email_with_attachment(sender, recipient, subject, body, attachment_path, aws_region): + # # Create a multipart/mixed parent container + # msg = MIMEMultipart('mixed') + # msg['Subject'] = subject + # msg['From'] = sender_email + # msg['To'] = recipient_email + + # # Add the text part + # text_part = MIMEText(body, 'plain') + # msg.attach(text_part) + + # # Add the attachment part + # with open(attachment_path, 'rb') as attachment_file: + # attachment_data = attachment_file.read() + # attachment_part = MIMEApplication(attachment_data) + # attachment_part.add_header('Content-Disposition', f'attachment; filename="{attachment_path}"') + # msg.attach(attachment_part) + + # # Send the email + # response = ses_client.send_raw_email( + # Source=sender, + # Destinations=[recipient], + # RawMessage={'Data': msg.as_string()} + # ) + + # ses_client.send_email( + # FromEmailAddress=settings.DEFAULT_FROM_EMAIL, + # Destination={"ToAddresses": [to_address]}, + # Content={ + # "Simple": { + # "Subject": {"Data": subject}, + # "Body": {"Text": {"Data": email_body}}, + # }, + # }, + # ) \ No newline at end of file From de17d686e3f73234c81915dabab88961e4f37a55 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:30:23 -0700 Subject: [PATCH 007/125] Add basic on hold modal --- src/registrar/assets/js/get-gov-admin.js | 2 +- .../django/admin/domain_change_form.html | 84 +++++++++++++++++-- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ff73acb65..096f7a626 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -29,7 +29,7 @@ function openInNewTab(el, removeAttribute = false){ */ (function (){ function createPhantomModalFormButtons(){ - let submitButtons = document.querySelectorAll('.usa-modal button[type="submit"]'); + let submitButtons = document.querySelectorAll('.usa-modal button[type="submit"].dja-form-placeholder'); form = document.querySelector("form") submitButtons.forEach((button) => { diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 67c5ac291..e38583eb8 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -11,18 +11,15 @@
{% if original.state != original.State.DELETED %} - + Extend expiration date | {% endif %} {% if original.state == original.State.READY %} - + + Place hold + {% elif original.state == original.State.ON_HOLD %} {% endif %} @@ -52,6 +49,8 @@ In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions of the application, so this means that it will briefly "populate", causing unintended visual effects. {% endcomment %} + + {# Create a modal for the _extend_expiration_date button #}
- +
+ + {# Create a modal for the _on_hold button #} +
+
+
+ +
+

+ When a domain is on hold: +

+
    +
  • The domain (and any subdomains) won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain will still appear in the registrar / admin.
  • +
  • Domain managers won’t be able to edit the domain.
  • +
+

+ This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State.ON_HOLD }} +

+
+ + +
+ +
+
+ {# Create a modal for when a domain is marked as ineligible #} {{ block.super }} {% endblock %} \ No newline at end of file From f693557f93eeb0d057d46a23ed80f79c4a3ea7b0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:58:40 -0700 Subject: [PATCH 008/125] Add some modals --- src/registrar/admin.py | 2 + .../admin/domain_application_change_form.html | 87 +++++++++++++++++++ .../django/admin/domain_change_form.html | 78 +++++++++++++++-- 3 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 src/registrar/templates/django/admin/domain_application_change_form.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 92e477667..942ae6162 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -931,6 +931,8 @@ class DomainApplicationAdmin(ListHeaderAdmin): if self.value() == "0": return queryset.filter(Q(is_election_board=False) | Q(is_election_board=None)) + change_form_template = "django/admin/domain_application_change_form.html" + # Columns list_display = [ "requested_domain", diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html new file mode 100644 index 000000000..26abbec18 --- /dev/null +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -0,0 +1,87 @@ +{% extends 'admin/change_form.html' %} +{% load i18n static %} + +{% block submit_buttons_bottom %} + {% comment %} + Modals behave very weirdly in django admin. + They tend to "strip out" any injected form elements, leaving only the main form. + In addition, USWDS handles modals by first destroying the element, then repopulating it toward the end of the page. + In effect, this means that the modal is not, and cannot, be surrounded by any form element at compile time. + + The current workaround for this is to use javascript to inject a hidden input, and bind submit of that + element to the click of the confirmation button within this modal. + + This is controlled by the class `dja-form-placeholder` on the button. + + In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions + of the application, so this means that it will briefly "populate", causing unintended visual effects. + {% endcomment %} +{# Create a modal for when a domain is marked as ineligible #} +
+
+
+ +
+

+ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: +

+
    +
  • They cannot edit the ineligible request or any other pending requests.
  • +
  • They cannot manage any of their approved domains.
  • +
  • They cannot initiate a new domain request.
  • +
+

+ This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State }} +

+
+ + +
+ +
+
+{{ block.super }} +{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index e38583eb8..9d7b1d5de 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -27,7 +27,9 @@ | {% endif %} {% if original.state != original.State.DELETED %} - + + Remove from registry + {% endif %} @@ -118,8 +120,8 @@
@@ -131,7 +133,7 @@ When a domain is on hold:

    -
  • The domain (and any subdomains) won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • The domain will still appear in the registrar / admin.
  • Domain managers won’t be able to edit the domain.
@@ -181,6 +183,72 @@
- {# Create a modal for when a domain is marked as ineligible #} + {# Create a modal for the _remove_domain button #} +
+
+
+ +
+

+ When a domain is removed from the registry: +

+
    +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain will still appear in the registrar / admin.
  • +
  • Domain managers won’t be able to edit the domain.
  • +
+

+ This action cannot be undone. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State.DELETED }} +

+
+ + +
+ +
+
{{ block.super }} {% endblock %} \ No newline at end of file From eeff5586d2b0de040ca2ed1614f94db0a9b3d618 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:20:33 -0700 Subject: [PATCH 009/125] Update styling --- src/registrar/assets/sass/_theme/_admin.scss | 9 +++++++++ .../django/admin/domain_change_form.html | 18 +++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index b57c6a015..46e4a10d4 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -302,3 +302,12 @@ input.admin-confirm-button { display: contents !important; } } + +.django-admin-modal .usa-prose ul > li { + list-style-type: inherit; + // Styling based off of the

styling in django admin + line-height: 1.5; + margin-bottom: 0; + margin-top: 0; + max-width: 68ex; +} diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 9d7b1d5de..61007c3d4 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -54,7 +54,7 @@ {# Create a modal for the _extend_expiration_date button #}

+ Domain: {{ original.name }} {# Acts as a
#}

- Domain: {{ original.name }} New status: {{ original.State.ON_HOLD }}

@@ -185,7 +185,7 @@
{# Create a modal for the _remove_domain button #}
    -
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • -
  • The domain will still appear in the registrar / admin.
  • -
  • Domain managers won’t be able to edit the domain.
  • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain will still appear in the registrar / admin.
  • +
  • Domain managers won’t be able to edit the domain.

This action cannot be undone.

+ Domain: {{ original.name }} {# Acts as a
#}

- Domain: {{ original.name }} New status: {{ original.State.DELETED }}

@@ -251,4 +251,4 @@ {{ block.super }} -{% endblock %} \ No newline at end of file +{% endblock %} From 8148b9099656e51ca26a2f514003d237eba29340 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 29 Feb 2024 09:06:58 -0800 Subject: [PATCH 010/125] Update documentation and add in the pyzipper import and associated biz logic --- .../runbooks/update_python_dependencies.md | 4 +- src/Pipfile | 2 + src/Pipfile.lock | 561 +++++++++--------- src/docker-compose.yml | 2 + src/registrar/config/settings.py | 3 + .../generate_current_metadata_report.py | 24 +- src/requirements.txt | 33 +- 7 files changed, 325 insertions(+), 304 deletions(-) diff --git a/docs/operations/runbooks/update_python_dependencies.md b/docs/operations/runbooks/update_python_dependencies.md index b94c0f39f..04fb936c6 100644 --- a/docs/operations/runbooks/update_python_dependencies.md +++ b/docs/operations/runbooks/update_python_dependencies.md @@ -2,7 +2,7 @@ ======================== 1. Check the [Pipfile](../../../src/Pipfile) for pinned dependencies and manually adjust the version numbers - +2. Run `docker-compose stop` to spin down the current containers and images so we can start afresh 2. Run cd src @@ -16,6 +16,6 @@ 3. Change geventconnpool back to what it was originally within the Pipfile.lock and requirements.txt. This is done by either saving what it was originally or opening a PR and using that as a reference to undo changes to any mention of geventconnpool. Geventconnpool, when set as a requirement without the reference portion, is defaulting to get a commit from 2014 which then breaks the code, as we want the newest version from them. -4. (optional) Run `docker-compose stop` and `docker-compose build` to build a new image for local development with the updated dependencies. +4. Run `docker-compose build` to build a new image for local development with the updated dependencies. The reason for de-coupling the `build` and `lock` steps is to increase consistency between builds--a run of `build` will always get exactly the dependencies listed in `Pipfile.lock`, nothing more, nothing less. \ No newline at end of file diff --git a/src/Pipfile b/src/Pipfile index 51417d578..8e43d1bab 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -30,6 +30,7 @@ greenlet = "*" gevent = "*" fred-epplib = {git = "https://github.com/cisagov/epplib.git", ref = "master"} geventconnpool = {git = "https://github.com/rasky/geventconnpool.git", ref = "1bbb93a714a331a069adf27265fe582d9ba7ecd4"} +pyzipper="*" [dev-packages] django-debug-toolbar = "*" @@ -45,3 +46,4 @@ types-cachetools = "*" boto3-mocking = "*" boto3-stubs = "*" django-model2puml = "*" + diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 7d511a0e5..c410630e1 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a672aeb8951fd850e90ad87c6f03cf71e2fc2b387d56fd3942361cb0b45bb449" + "sha256": "8c15011f6c6e0447e4ca675ce840fe6b67048e90255e7c083be357b373f96a47" }, "pipfile-spec": 6, "requires": {}, @@ -32,29 +32,29 @@ }, "boto3": { "hashes": [ - "sha256:65acfe7f1cf2a9b7df3d4edb87c8022e02685825bd1957e7bb678cc0d09f5e5f", - "sha256:73f5ec89cb3ddb3ed577317889fd2f2df783f66b6502a9a4239979607e33bf74" + "sha256:66303b5f26d92afb72656ff490b22ea72dfff8bf1a29e4a0c5d5f11ec56245dd", + "sha256:898ad2123b18cae8efd85adc56ac2d1925be54592aebc237020d4f16e9a9e7a9" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.37" + "version": "==1.34.52" }, "botocore": { "hashes": [ - "sha256:2a5bf33aacd2d970afd3d492e179e06ea98a5469030d5cfe7a2ad9995f7bb2ef", - "sha256:3c46ddb1679e6ef45ca78b48665398636bda532a07cd476e4b500697d13d9a99" + "sha256:05567d8aba344826060481ea309555432c96f0febe22bee7cf5a3b6d3a03cec8", + "sha256:187da93aec3f2e87d8a31eced16fa2cb9c71fe2d69b0a797f9f7a9220f5bf7ae" ], "markers": "python_version >= '3.8'", - "version": "==1.34.37" + "version": "==1.34.52" }, "cachetools": { "hashes": [ - "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2", - "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1" + "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945", + "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==5.3.2" + "version": "==5.3.3" }, "certifi": { "hashes": [ @@ -228,41 +228,41 @@ }, "cryptography": { "hashes": [ - "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380", - "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589", - "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea", - "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65", - "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a", - "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3", - "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008", - "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1", - "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2", - "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635", - "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2", - "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90", - "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee", - "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a", - "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242", - "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12", - "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2", - "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d", - "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be", - "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee", - "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6", - "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529", - "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929", - "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1", - "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6", - "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a", - "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446", - "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9", - "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888", - "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4", - "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33", - "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f" + "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee", + "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576", + "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d", + "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30", + "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413", + "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb", + "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da", + "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4", + "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd", + "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc", + "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8", + "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1", + "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc", + "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e", + "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8", + "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940", + "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400", + "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7", + "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16", + "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278", + "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74", + "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec", + "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1", + "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2", + "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c", + "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922", + "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a", + "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6", + "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1", + "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e", + "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac", + "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7" ], "markers": "python_version >= '3.7'", - "version": "==42.0.2" + "version": "==42.0.5" }, "defusedxml": { "hashes": [ @@ -384,12 +384,12 @@ }, "faker": { "hashes": [ - "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2", - "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b" + "sha256:117ce1a2805c1bc5ca753b3dc6f9d567732893b2294b827d3164261ee8f20267", + "sha256:458d93580de34403a8dec1e8d5e6be2fee96c4deca63b95d71df7a6a80a690de" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==23.1.0" + "version": "==23.3.0" }, "fred-epplib": { "git": "https://github.com/cisagov/epplib.git", @@ -404,61 +404,63 @@ }, "future": { "hashes": [ - "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307" + "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", + "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05" ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.18.3" + "version": "==1.0.0" }, "gevent": { "hashes": [ - "sha256:272cffdf535978d59c38ed837916dfd2b5d193be1e9e5dcc60a5f4d5025dd98a", - "sha256:2c7b5c9912378e5f5ccf180d1fdb1e83f42b71823483066eddbe10ef1a2fcaa2", - "sha256:36a549d632c14684bcbbd3014a6ce2666c5f2a500f34d58d32df6c9ea38b6535", - "sha256:4368f341a5f51611411ec3fc62426f52ac3d6d42eaee9ed0f9eebe715c80184e", - "sha256:43daf68496c03a35287b8b617f9f91e0e7c0d042aebcc060cadc3f049aadd653", - "sha256:455e5ee8103f722b503fa45dedb04f3ffdec978c1524647f8ba72b4f08490af1", - "sha256:45792c45d60f6ce3d19651d7fde0bc13e01b56bb4db60d3f32ab7d9ec467374c", - "sha256:4e24c2af9638d6c989caffc691a039d7c7022a31c0363da367c0d32ceb4a0648", - "sha256:52b4abf28e837f1865a9bdeef58ff6afd07d1d888b70b6804557e7908032e599", - "sha256:52e9f12cd1cda96603ce6b113d934f1aafb873e2c13182cf8e86d2c5c41982ea", - "sha256:5f3c781c84794926d853d6fb58554dc0dcc800ba25c41d42f6959c344b4db5a6", - "sha256:62d121344f7465e3739989ad6b91f53a6ca9110518231553fe5846dbe1b4518f", - "sha256:65883ac026731ac112184680d1f0f1e39fa6f4389fd1fc0bf46cc1388e2599f9", - "sha256:707904027d7130ff3e59ea387dddceedb133cc742b00b3ffe696d567147a9c9e", - "sha256:72c002235390d46f94938a96920d8856d4ffd9ddf62a303a0d7c118894097e34", - "sha256:7532c17bc6c1cbac265e751b95000961715adef35a25d2b0b1813aa7263fb397", - "sha256:78eebaf5e73ff91d34df48f4e35581ab4c84e22dd5338ef32714264063c57507", - "sha256:7c1abc6f25f475adc33e5fc2dbcc26a732608ac5375d0d306228738a9ae14d3b", - "sha256:7c28e38dcde327c217fdafb9d5d17d3e772f636f35df15ffae2d933a5587addd", - "sha256:7ccf0fd378257cb77d91c116e15c99e533374a8153632c48a3ecae7f7f4f09fe", - "sha256:921dda1c0b84e3d3b1778efa362d61ed29e2b215b90f81d498eb4d8eafcd0b7a", - "sha256:a2898b7048771917d85a1d548fd378e8a7b2ca963db8e17c6d90c76b495e0e2b", - "sha256:a3c5e9b1f766a7a64833334a18539a362fb563f6c4682f9634dea72cbe24f771", - "sha256:ada07076b380918829250201df1d016bdafb3acf352f35e5693b59dceee8dd2e", - "sha256:b101086f109168b23fa3586fccd1133494bdb97f86920a24dc0b23984dc30b69", - "sha256:bf456bd6b992eb0e1e869e2fd0caf817f0253e55ca7977fd0e72d0336a8c1c6a", - "sha256:bf7af500da05363e66f122896012acb6e101a552682f2352b618e541c941a011", - "sha256:c3e5d2fa532e4d3450595244de8ccf51f5721a05088813c1abd93ad274fe15e7", - "sha256:c84d34256c243b0a53d4335ef0bc76c735873986d478c53073861a92566a8d71", - "sha256:d163d59f1be5a4c4efcdd13c2177baaf24aadf721fdf2e1af9ee54a998d160f5", - "sha256:d57737860bfc332b9b5aa438963986afe90f49645f6e053140cfa0fa1bdae1ae", - "sha256:dbb22a9bbd6a13e925815ce70b940d1578dbe5d4013f20d23e8a11eddf8d14a7", - "sha256:dcb8612787a7f4626aa881ff15ff25439561a429f5b303048f0fca8a1c781c39", - "sha256:dd6c32ab977ecf7c7b8c2611ed95fa4aaebd69b74bf08f4b4960ad516861517d", - "sha256:de350fde10efa87ea60d742901e1053eb2127ebd8b59a7d3b90597eb4e586599", - "sha256:e1ead6863e596a8cc2a03e26a7a0981f84b6b3e956101135ff6d02df4d9a6b07", - "sha256:ed7a048d3e526a5c1d55c44cb3bc06cfdc1947d06d45006cc4cf60dedc628904", - "sha256:f632487c87866094546a74eefbca2c74c1d03638b715b6feb12e80120960185a", - "sha256:fae8d5b5b8fa2a8f63b39f5447168b02db10c888a3e387ed7af2bd1b8612e543", - "sha256:fde6402c5432b835fbb7698f1c7f2809c8d6b2bd9d047ac1f5a7c1d5aa569303" + "sha256:03aa5879acd6b7076f6a2a307410fb1e0d288b84b03cdfd8c74db8b4bc882fc5", + "sha256:117e5837bc74a1673605fb53f8bfe22feb6e5afa411f524c835b2ddf768db0de", + "sha256:141a2b24ad14f7b9576965c0c84927fc85f824a9bb19f6ec1e61e845d87c9cd8", + "sha256:14532a67f7cb29fb055a0e9b39f16b88ed22c66b96641df8c04bdc38c26b9ea5", + "sha256:1dffb395e500613e0452b9503153f8f7ba587c67dd4a85fc7cd7aa7430cb02cc", + "sha256:2955eea9c44c842c626feebf4459c42ce168685aa99594e049d03bedf53c2800", + "sha256:2ae3a25ecce0a5b0cd0808ab716bfca180230112bb4bc89b46ae0061d62d4afe", + "sha256:2e9ac06f225b696cdedbb22f9e805e2dd87bf82e8fa5e17756f94e88a9d37cf7", + "sha256:368a277bd9278ddb0fde308e6a43f544222d76ed0c4166e0d9f6b036586819d9", + "sha256:3adfb96637f44010be8abd1b5e73b5070f851b817a0b182e601202f20fa06533", + "sha256:3d5325ccfadfd3dcf72ff88a92fb8fc0b56cacc7225f0f4b6dcf186c1a6eeabc", + "sha256:432fc76f680acf7cf188c2ee0f5d3ab73b63c1f03114c7cd8a34cebbe5aa2056", + "sha256:44098038d5e2749b0784aabb27f1fcbb3f43edebedf64d0af0d26955611be8d6", + "sha256:5a1df555431f5cd5cc189a6ee3544d24f8c52f2529134685f1e878c4972ab026", + "sha256:6c47ae7d1174617b3509f5d884935e788f325eb8f1a7efc95d295c68d83cce40", + "sha256:6f947a9abc1a129858391b3d9334c45041c08a0f23d14333d5b844b6e5c17a07", + "sha256:782a771424fe74bc7e75c228a1da671578c2ba4ddb2ca09b8f959abdf787331e", + "sha256:7899a38d0ae7e817e99adb217f586d0a4620e315e4de577444ebeeed2c5729be", + "sha256:7b00f8c9065de3ad226f7979154a7b27f3b9151c8055c162332369262fc025d8", + "sha256:8f4b8e777d39013595a7740b4463e61b1cfe5f462f1b609b28fbc1e4c4ff01e5", + "sha256:90cbac1ec05b305a1b90ede61ef73126afdeb5a804ae04480d6da12c56378df1", + "sha256:918cdf8751b24986f915d743225ad6b702f83e1106e08a63b736e3a4c6ead789", + "sha256:9202f22ef811053077d01f43cc02b4aaf4472792f9fd0f5081b0b05c926cca19", + "sha256:94138682e68ec197db42ad7442d3cf9b328069c3ad8e4e5022e6b5cd3e7ffae5", + "sha256:968581d1717bbcf170758580f5f97a2925854943c45a19be4d47299507db2eb7", + "sha256:9d8d0642c63d453179058abc4143e30718b19a85cbf58c2744c9a63f06a1d388", + "sha256:a7ceb59986456ce851160867ce4929edaffbd2f069ae25717150199f8e1548b8", + "sha256:b9913c45d1be52d7a5db0c63977eebb51f68a2d5e6fd922d1d9b5e5fd758cc98", + "sha256:bde283313daf0b34a8d1bab30325f5cb0f4e11b5869dbe5bc61f8fe09a8f66f3", + "sha256:bf5b9c72b884c6f0c4ed26ef204ee1f768b9437330422492c319470954bc4cc7", + "sha256:ca80b121bbec76d7794fcb45e65a7eca660a76cc1a104ed439cdbd7df5f0b060", + "sha256:cdf66977a976d6a3cfb006afdf825d1482f84f7b81179db33941f2fc9673bb1d", + "sha256:d4faf846ed132fd7ebfbbf4fde588a62d21faa0faa06e6f468b7faa6f436b661", + "sha256:d7f87c2c02e03d99b95cfa6f7a776409083a9e4d468912e18c7680437b29222c", + "sha256:dd23df885318391856415e20acfd51a985cba6919f0be78ed89f5db9ff3a31cb", + "sha256:f5de3c676e57177b38857f6e3cdfbe8f38d1cd754b63200c0615eaa31f514b4f", + "sha256:f5e8e8d60e18d5f7fd49983f0c4696deeddaf6e608fbab33397671e2fcc6cc91", + "sha256:f7cac622e11b4253ac4536a654fe221249065d9a69feb6cdcd4d9af3503602e0", + "sha256:f8a04cf0c5b7139bc6368b461257d4a757ea2fe89b3773e494d235b7dd51119f", + "sha256:f8bb35ce57a63c9a6896c71a285818a3922d8ca05d150fd1fe49a7f57287b836", + "sha256:fbfdce91239fe306772faab57597186710d5699213f4df099d1612da7320d682" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==23.9.1" + "version": "==24.2.1" }, "geventconnpool": { "git": "https://github.com/rasky/geventconnpool.git", - "ref": "1bbb93a714a331a069adf27265fe582d9ba7ecd4" + "ref": null }, "greenlet": { "hashes": [ @@ -710,11 +712,11 @@ }, "marshmallow": { "hashes": [ - "sha256:4c1daff273513dc5eb24b219a8035559dc573c8f322558ef85f5438ddd1236dd", - "sha256:c21d4b98fee747c130e6bc8f45c4b3199ea66bc00c12ee1f639f0aeca034d5e9" + "sha256:20f53be28c6e374a711a16165fb22a8dc6003e3f7cda1285e3ca777b9193885b", + "sha256:e7997f83571c7fd476042c2c188e4ee8a78900ca5e74bd9c8097afa56624e9bd" ], "markers": "python_version >= '3.8'", - "version": "==3.20.2" + "version": "==3.21.0" }, "oic": { "hashes": [ @@ -742,10 +744,10 @@ }, "phonenumberslite": { "hashes": [ - "sha256:2b04a53401d01ab42564c1abc762fc9808ad398e71dacfa3b38d4321e112ecb3", - "sha256:74e3ee63dfa2bb562ce2e6ce74ce76ae74a2f81472005b80343235fb43426db4" + "sha256:137d53d5d78dca30bc2becf81a3e2ac74deb8f0997e9bbe44de515ece4bd92bd", + "sha256:e1f4359bff90c86d1b52db0e726d3334df00cc7d9c9c2ef66561d5f7a774d4ba" ], - "version": "==8.13.29" + "version": "==8.13.31" }, "psycopg2-binary": { "hashes": [ @@ -874,104 +876,104 @@ }, "pydantic": { "hashes": [ - "sha256:0b6a909df3192245cb736509a92ff69e4fef76116feffec68e93a567347bae6f", - "sha256:4fd5c182a2488dc63e6d32737ff19937888001e2a6d86e94b3f233104a5d1fa9" + "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a", + "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f" ], "markers": "python_version >= '3.8'", - "version": "==2.6.1" + "version": "==2.6.3" }, "pydantic-core": { "hashes": [ - "sha256:02906e7306cb8c5901a1feb61f9ab5e5c690dbbeaa04d84c1b9ae2a01ebe9379", - "sha256:0ba503850d8b8dcc18391f10de896ae51d37fe5fe43dbfb6a35c5c5cad271a06", - "sha256:16aa02e7a0f539098e215fc193c8926c897175d64c7926d00a36188917717a05", - "sha256:18de31781cdc7e7b28678df7c2d7882f9692ad060bc6ee3c94eb15a5d733f8f7", - "sha256:22c5f022799f3cd6741e24f0443ead92ef42be93ffda0d29b2597208c94c3753", - "sha256:2924b89b16420712e9bb8192396026a8fbd6d8726224f918353ac19c4c043d2a", - "sha256:308974fdf98046db28440eb3377abba274808bf66262e042c412eb2adf852731", - "sha256:396fdf88b1b503c9c59c84a08b6833ec0c3b5ad1a83230252a9e17b7dfb4cffc", - "sha256:3ac426704840877a285d03a445e162eb258924f014e2f074e209d9b4ff7bf380", - "sha256:3b052c753c4babf2d1edc034c97851f867c87d6f3ea63a12e2700f159f5c41c3", - "sha256:3fab4e75b8c525a4776e7630b9ee48aea50107fea6ca9f593c98da3f4d11bf7c", - "sha256:406fac1d09edc613020ce9cf3f2ccf1a1b2f57ab00552b4c18e3d5276c67eb11", - "sha256:40a0bd0bed96dae5712dab2aba7d334a6c67cbcac2ddfca7dbcc4a8176445990", - "sha256:41dac3b9fce187a25c6253ec79a3f9e2a7e761eb08690e90415069ea4a68ff7a", - "sha256:459c0d338cc55d099798618f714b21b7ece17eb1a87879f2da20a3ff4c7628e2", - "sha256:459d6be6134ce3b38e0ef76f8a672924460c455d45f1ad8fdade36796df1ddc8", - "sha256:46b0d5520dbcafea9a8645a8164658777686c5c524d381d983317d29687cce97", - "sha256:47924039e785a04d4a4fa49455e51b4eb3422d6eaacfde9fc9abf8fdef164e8a", - "sha256:4bfcbde6e06c56b30668a0c872d75a7ef3025dc3c1823a13cf29a0e9b33f67e8", - "sha256:4f9ee4febb249c591d07b2d4dd36ebcad0ccd128962aaa1801508320896575ef", - "sha256:55749f745ebf154c0d63d46c8c58594d8894b161928aa41adbb0709c1fe78b77", - "sha256:5864b0242f74b9dd0b78fd39db1768bc3f00d1ffc14e596fd3e3f2ce43436a33", - "sha256:5f60f920691a620b03082692c378661947d09415743e437a7478c309eb0e4f82", - "sha256:60eb8ceaa40a41540b9acae6ae7c1f0a67d233c40dc4359c256ad2ad85bdf5e5", - "sha256:69a7b96b59322a81c2203be537957313b07dd333105b73db0b69212c7d867b4b", - "sha256:6ad84731a26bcfb299f9eab56c7932d46f9cad51c52768cace09e92a19e4cf55", - "sha256:6db58c22ac6c81aeac33912fb1af0e930bc9774166cdd56eade913d5f2fff35e", - "sha256:70651ff6e663428cea902dac297066d5c6e5423fda345a4ca62430575364d62b", - "sha256:72f7919af5de5ecfaf1eba47bf9a5d8aa089a3340277276e5636d16ee97614d7", - "sha256:732bd062c9e5d9582a30e8751461c1917dd1ccbdd6cafb032f02c86b20d2e7ec", - "sha256:7924e54f7ce5d253d6160090ddc6df25ed2feea25bfb3339b424a9dd591688bc", - "sha256:7afb844041e707ac9ad9acad2188a90bffce2c770e6dc2318be0c9916aef1469", - "sha256:7b883af50eaa6bb3299780651e5be921e88050ccf00e3e583b1e92020333304b", - "sha256:7beec26729d496a12fd23cf8da9944ee338c8b8a17035a560b585c36fe81af20", - "sha256:7bf26c2e2ea59d32807081ad51968133af3025c4ba5753e6a794683d2c91bf6e", - "sha256:7c31669e0c8cc68400ef0c730c3a1e11317ba76b892deeefaf52dcb41d56ed5d", - "sha256:7e6231aa5bdacda78e96ad7b07d0c312f34ba35d717115f4b4bff6cb87224f0f", - "sha256:870dbfa94de9b8866b37b867a2cb37a60c401d9deb4a9ea392abf11a1f98037b", - "sha256:88646cae28eb1dd5cd1e09605680c2b043b64d7481cdad7f5003ebef401a3039", - "sha256:8aafeedb6597a163a9c9727d8a8bd363a93277701b7bfd2749fbefee2396469e", - "sha256:8bde5b48c65b8e807409e6f20baee5d2cd880e0fad00b1a811ebc43e39a00ab2", - "sha256:8f9142a6ed83d90c94a3efd7af8873bf7cefed2d3d44387bf848888482e2d25f", - "sha256:936a787f83db1f2115ee829dd615c4f684ee48ac4de5779ab4300994d8af325b", - "sha256:98dc6f4f2095fc7ad277782a7c2c88296badcad92316b5a6e530930b1d475ebc", - "sha256:9957433c3a1b67bdd4c63717eaf174ebb749510d5ea612cd4e83f2d9142f3fc8", - "sha256:99af961d72ac731aae2a1b55ccbdae0733d816f8bfb97b41909e143de735f522", - "sha256:9b5f13857da99325dcabe1cc4e9e6a3d7b2e2c726248ba5dd4be3e8e4a0b6d0e", - "sha256:9d776d30cde7e541b8180103c3f294ef7c1862fd45d81738d156d00551005784", - "sha256:9da90d393a8227d717c19f5397688a38635afec89f2e2d7af0df037f3249c39a", - "sha256:a3b7352b48fbc8b446b75f3069124e87f599d25afb8baa96a550256c031bb890", - "sha256:a477932664d9611d7a0816cc3c0eb1f8856f8a42435488280dfbf4395e141485", - "sha256:a7e41e3ada4cca5f22b478c08e973c930e5e6c7ba3588fb8e35f2398cdcc1545", - "sha256:a90fec23b4b05a09ad988e7a4f4e081711a90eb2a55b9c984d8b74597599180f", - "sha256:a9e523474998fb33f7c1a4d55f5504c908d57add624599e095c20fa575b8d943", - "sha256:aa057095f621dad24a1e906747179a69780ef45cc8f69e97463692adbcdae878", - "sha256:aa6c8c582036275997a733427b88031a32ffa5dfc3124dc25a730658c47a572f", - "sha256:ae34418b6b389d601b31153b84dce480351a352e0bb763684a1b993d6be30f17", - "sha256:b0d7a9165167269758145756db43a133608a531b1e5bb6a626b9ee24bc38a8f7", - "sha256:b30b0dd58a4509c3bd7eefddf6338565c4905406aee0c6e4a5293841411a1286", - "sha256:b8f9186ca45aee030dc8234118b9c0784ad91a0bb27fc4e7d9d6608a5e3d386c", - "sha256:b94cbda27267423411c928208e89adddf2ea5dd5f74b9528513f0358bba019cb", - "sha256:cc6f6c9be0ab6da37bc77c2dda5f14b1d532d5dbef00311ee6e13357a418e646", - "sha256:ce232a6170dd6532096cadbf6185271e4e8c70fc9217ebe105923ac105da9978", - "sha256:cf903310a34e14651c9de056fcc12ce090560864d5a2bb0174b971685684e1d8", - "sha256:d5362d099c244a2d2f9659fb3c9db7c735f0004765bbe06b99be69fbd87c3f15", - "sha256:dffaf740fe2e147fedcb6b561353a16243e654f7fe8e701b1b9db148242e1272", - "sha256:e0f686549e32ccdb02ae6f25eee40cc33900910085de6aa3790effd391ae10c2", - "sha256:e4b52776a2e3230f4854907a1e0946eec04d41b1fc64069ee774876bbe0eab55", - "sha256:e4ba0884a91f1aecce75202473ab138724aa4fb26d7707f2e1fa6c3e68c84fbf", - "sha256:e6294e76b0380bb7a61eb8a39273c40b20beb35e8c87ee101062834ced19c545", - "sha256:ebb892ed8599b23fa8f1799e13a12c87a97a6c9d0f497525ce9858564c4575a4", - "sha256:eca58e319f4fd6df004762419612122b2c7e7d95ffafc37e890252f869f3fb2a", - "sha256:ed957db4c33bc99895f3a1672eca7e80e8cda8bd1e29a80536b4ec2153fa9804", - "sha256:ef551c053692b1e39e3f7950ce2296536728871110e7d75c4e7753fb30ca87f4", - "sha256:ef6113cd31411eaf9b39fc5a8848e71c72656fd418882488598758b2c8c6dfa0", - "sha256:f685dbc1fdadb1dcd5b5e51e0a378d4685a891b2ddaf8e2bba89bd3a7144e44a", - "sha256:f8ed79883b4328b7f0bd142733d99c8e6b22703e908ec63d930b06be3a0e7113", - "sha256:fe56851c3f1d6f5384b3051c536cc81b3a93a73faf931f404fef95217cf1e10d", - "sha256:ff7c97eb7a29aba230389a2661edf2e9e06ce616c7e35aa764879b6894a44b25" + "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a", + "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed", + "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979", + "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff", + "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5", + "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45", + "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340", + "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad", + "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23", + "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6", + "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7", + "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241", + "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda", + "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187", + "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba", + "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c", + "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2", + "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c", + "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132", + "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf", + "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972", + "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db", + "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade", + "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4", + "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8", + "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f", + "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9", + "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48", + "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec", + "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d", + "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9", + "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb", + "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4", + "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89", + "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c", + "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9", + "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da", + "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac", + "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b", + "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf", + "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e", + "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137", + "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1", + "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b", + "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8", + "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e", + "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053", + "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01", + "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe", + "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd", + "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805", + "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183", + "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8", + "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99", + "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820", + "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074", + "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256", + "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8", + "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975", + "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad", + "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e", + "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca", + "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df", + "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b", + "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a", + "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a", + "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721", + "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a", + "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f", + "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2", + "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97", + "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6", + "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed", + "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc", + "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1", + "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe", + "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120", + "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f", + "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a" ], "markers": "python_version >= '3.8'", - "version": "==2.16.2" + "version": "==2.16.3" }, "pydantic-settings": { "hashes": [ - "sha256:26b1492e0a24755626ac5e6d715e9077ab7ad4fb5f19a8b7ed7011d52f36141c", - "sha256:7621c0cb5d90d1140d2f0ef557bdf03573aac7035948109adf2574770b77605a" + "sha256:00b9f6a5e95553590434c0fa01ead0b216c3e10bc54ae02e37f359948643c5ed", + "sha256:0235391d26db4d2190cb9b31051c4b46882d28a51533f97440867f012d4da091" ], "markers": "python_version >= '3.8'", - "version": "==2.1.0" + "version": "==2.2.1" }, "pyjwkest": { "hashes": [ @@ -996,6 +998,15 @@ "markers": "python_version >= '3.8'", "version": "==1.0.1" }, + "pyzipper": { + "hashes": [ + "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc", + "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87" + ], + "index": "pypi", + "markers": "python_version >= '3.4'", + "version": "==0.3.6" + }, "requests": { "hashes": [ "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", @@ -1015,11 +1026,11 @@ }, "setuptools": { "hashes": [ - "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05", - "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78" + "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", + "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" ], "markers": "python_version >= '3.8'", - "version": "==69.0.3" + "version": "==69.1.1" }, "six": { "hashes": [ @@ -1039,12 +1050,12 @@ }, "typing-extensions": { "hashes": [ - "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", - "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", + "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.9.0" + "version": "==4.10.0" }, "urllib3": { "hashes": [ @@ -1073,45 +1084,45 @@ }, "zope.interface": { "hashes": [ - "sha256:0c8cf55261e15590065039696607f6c9c1aeda700ceee40c70478552d323b3ff", - "sha256:13b7d0f2a67eb83c385880489dbb80145e9d344427b4262c49fbf2581677c11c", - "sha256:1f294a15f7723fc0d3b40701ca9b446133ec713eafc1cc6afa7b3d98666ee1ac", - "sha256:239a4a08525c080ff833560171d23b249f7f4d17fcbf9316ef4159f44997616f", - "sha256:2f8d89721834524a813f37fa174bac074ec3d179858e4ad1b7efd4401f8ac45d", - "sha256:2fdc7ccbd6eb6b7df5353012fbed6c3c5d04ceaca0038f75e601060e95345309", - "sha256:34c15ca9248f2e095ef2e93af2d633358c5f048c49fbfddf5fdfc47d5e263736", - "sha256:387545206c56b0315fbadb0431d5129c797f92dc59e276b3ce82db07ac1c6179", - "sha256:43b576c34ef0c1f5a4981163b551a8781896f2a37f71b8655fd20b5af0386abb", - "sha256:57d0a8ce40ce440f96a2c77824ee94bf0d0925e6089df7366c2272ccefcb7941", - "sha256:5a804abc126b33824a44a7aa94f06cd211a18bbf31898ba04bd0924fbe9d282d", - "sha256:67be3ca75012c6e9b109860820a8b6c9a84bfb036fbd1076246b98e56951ca92", - "sha256:6af47f10cfc54c2ba2d825220f180cc1e2d4914d783d6fc0cd93d43d7bc1c78b", - "sha256:6dc998f6de015723196a904045e5a2217f3590b62ea31990672e31fbc5370b41", - "sha256:70d2cef1bf529bff41559be2de9d44d47b002f65e17f43c73ddefc92f32bf00f", - "sha256:7ebc4d34e7620c4f0da7bf162c81978fce0ea820e4fa1e8fc40ee763839805f3", - "sha256:964a7af27379ff4357dad1256d9f215047e70e93009e532d36dcb8909036033d", - "sha256:97806e9ca3651588c1baaebb8d0c5ee3db95430b612db354c199b57378312ee8", - "sha256:9b9bc671626281f6045ad61d93a60f52fd5e8209b1610972cf0ef1bbe6d808e3", - "sha256:9ffdaa5290422ac0f1688cb8adb1b94ca56cee3ad11f29f2ae301df8aecba7d1", - "sha256:a0da79117952a9a41253696ed3e8b560a425197d4e41634a23b1507efe3273f1", - "sha256:a41f87bb93b8048fe866fa9e3d0c51e27fe55149035dcf5f43da4b56732c0a40", - "sha256:aa6fd016e9644406d0a61313e50348c706e911dca29736a3266fc9e28ec4ca6d", - "sha256:ad54ed57bdfa3254d23ae04a4b1ce405954969c1b0550cc2d1d2990e8b439de1", - "sha256:b012d023b4fb59183909b45d7f97fb493ef7a46d2838a5e716e3155081894605", - "sha256:b51b64432eed4c0744241e9ce5c70dcfecac866dff720e746d0a9c82f371dfa7", - "sha256:bbe81def9cf3e46f16ce01d9bfd8bea595e06505e51b7baf45115c77352675fd", - "sha256:c9559138690e1bd4ea6cd0954d22d1e9251e8025ce9ede5d0af0ceae4a401e43", - "sha256:e30506bcb03de8983f78884807e4fd95d8db6e65b69257eea05d13d519b83ac0", - "sha256:e33e86fd65f369f10608b08729c8f1c92ec7e0e485964670b4d2633a4812d36b", - "sha256:e441e8b7d587af0414d25e8d05e27040d78581388eed4c54c30c0c91aad3a379", - "sha256:e8bb9c990ca9027b4214fa543fd4025818dc95f8b7abce79d61dc8a2112b561a", - "sha256:ef43ee91c193f827e49599e824385ec7c7f3cd152d74cb1dfe02cb135f264d83", - "sha256:ef467d86d3cfde8b39ea1b35090208b0447caaabd38405420830f7fd85fbdd56", - "sha256:f89b28772fc2562ed9ad871c865f5320ef761a7fcc188a935e21fe8b31a38ca9", - "sha256:fddbab55a2473f1d3b8833ec6b7ac31e8211b0aa608df5ab09ce07f3727326de" + "sha256:02adbab560683c4eca3789cc0ac487dcc5f5a81cc48695ec247f00803cafe2fe", + "sha256:14e02a6fc1772b458ebb6be1c276528b362041217b9ca37e52ecea2cbdce9fac", + "sha256:25e0af9663eeac6b61b231b43c52293c2cb7f0c232d914bdcbfd3e3bd5c182ad", + "sha256:2606955a06c6852a6cff4abeca38346ed01e83f11e960caa9a821b3626a4467b", + "sha256:396f5c94654301819a7f3a702c5830f0ea7468d7b154d124ceac823e2419d000", + "sha256:3b240883fb43160574f8f738e6d09ddbdbf8fa3e8cea051603d9edfd947d9328", + "sha256:3b6c62813c63c543a06394a636978b22dffa8c5410affc9331ce6cdb5bfa8565", + "sha256:4ae9793f114cee5c464cc0b821ae4d36e1eba961542c6086f391a61aee167b6f", + "sha256:4bce517b85f5debe07b186fc7102b332676760f2e0c92b7185dd49c138734b70", + "sha256:4d45d2ba8195850e3e829f1f0016066a122bfa362cc9dc212527fc3d51369037", + "sha256:4dd374927c00764fcd6fe1046bea243ebdf403fba97a937493ae4be2c8912c2b", + "sha256:506f5410b36e5ba494136d9fa04c548eaf1a0d9c442b0b0e7a0944db7620e0ab", + "sha256:59f7374769b326a217d0b2366f1c176a45a4ff21e8f7cebb3b4a3537077eff85", + "sha256:5ee9789a20b0081dc469f65ff6c5007e67a940d5541419ca03ef20c6213dd099", + "sha256:6fc711acc4a1c702ca931fdbf7bf7c86f2a27d564c85c4964772dadf0e3c52f5", + "sha256:75d2ec3d9b401df759b87bc9e19d1b24db73083147089b43ae748aefa63067ef", + "sha256:76e0531d86523be7a46e15d379b0e975a9db84316617c0efe4af8338dc45b80c", + "sha256:8af82afc5998e1f307d5e72712526dba07403c73a9e287d906a8aa2b1f2e33dd", + "sha256:8f5d2c39f3283e461de3655e03faf10e4742bb87387113f787a7724f32db1e48", + "sha256:97785604824981ec8c81850dd25c8071d5ce04717a34296eeac771231fbdd5cd", + "sha256:a3046e8ab29b590d723821d0785598e0b2e32b636a0272a38409be43e3ae0550", + "sha256:abb0b3f2cb606981c7432f690db23506b1db5899620ad274e29dbbbdd740e797", + "sha256:ac7c2046d907e3b4e2605a130d162b1b783c170292a11216479bb1deb7cadebe", + "sha256:af27b3fe5b6bf9cd01b8e1c5ddea0a0d0a1b8c37dc1c7452f1e90bf817539c6d", + "sha256:b386b8b9d2b6a5e1e4eadd4e62335571244cb9193b7328c2b6e38b64cfda4f0e", + "sha256:b66335bbdbb4c004c25ae01cc4a54fd199afbc1fd164233813c6d3c2293bb7e1", + "sha256:d54f66c511ea01b9ef1d1a57420a93fbb9d48a08ec239f7d9c581092033156d0", + "sha256:de125151a53ecdb39df3cb3deb9951ed834dd6a110a9e795d985b10bb6db4532", + "sha256:de7916380abaef4bb4891740879b1afcba2045aee51799dfd6d6ca9bdc71f35f", + "sha256:e2fefad268ff5c5b314794e27e359e48aeb9c8bb2cbb5748a071757a56f6bb8f", + "sha256:e7b2bed4eea047a949296e618552d3fed00632dc1b795ee430289bdd0e3717f3", + "sha256:e87698e2fea5ca2f0a99dff0a64ce8110ea857b640de536c76d92aaa2a91ff3a", + "sha256:ede888382882f07b9e4cd942255921ffd9f2901684198b88e247c7eabd27a000", + "sha256:f444de0565db46d26c9fa931ca14f497900a295bd5eba480fc3fad25af8c763e", + "sha256:fa994e8937e8ccc7e87395b7b35092818905cf27c651e3ff3e7f29729f5ce3ce", + "sha256:febceb04ee7dd2aef08c2ff3d6f8a07de3052fc90137c507b0ede3ea80c21440" ], "markers": "python_version >= '3.7'", - "version": "==6.1" + "version": "==6.2" } }, "develop": { @@ -1142,32 +1153,32 @@ }, "black": { "hashes": [ - "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8", - "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6", - "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62", - "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445", - "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c", - "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a", - "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9", - "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2", - "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6", - "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b", - "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4", - "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168", - "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d", - "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5", - "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024", - "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e", - "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b", - "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161", - "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717", - "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8", - "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac", - "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7" + "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8", + "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8", + "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd", + "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9", + "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31", + "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92", + "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f", + "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29", + "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4", + "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693", + "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218", + "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a", + "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23", + "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0", + "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982", + "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894", + "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540", + "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430", + "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b", + "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2", + "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", + "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==24.1.1" + "version": "==24.2.0" }, "blinker": { "hashes": [ @@ -1179,12 +1190,12 @@ }, "boto3": { "hashes": [ - "sha256:65acfe7f1cf2a9b7df3d4edb87c8022e02685825bd1957e7bb678cc0d09f5e5f", - "sha256:73f5ec89cb3ddb3ed577317889fd2f2df783f66b6502a9a4239979607e33bf74" + "sha256:66303b5f26d92afb72656ff490b22ea72dfff8bf1a29e4a0c5d5f11ec56245dd", + "sha256:898ad2123b18cae8efd85adc56ac2d1925be54592aebc237020d4f16e9a9e7a9" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.37" + "version": "==1.34.52" }, "boto3-mocking": { "hashes": [ @@ -1197,28 +1208,28 @@ }, "boto3-stubs": { "hashes": [ - "sha256:97b5ca3d3145385acde5af46ca2da3fc74f433545034c36183f389e99771516e", - "sha256:c6618c7126bac0337c05e161e9c428febc57d6a24d7ff62de46e67761f402c57" + "sha256:644381a404fb5884154f7dcc40bb819f0c7f37de21b7a7b493585277b51c9a5f", + "sha256:823c41059f836d6877daaa1cbd20f813c8f1a78b9fdf290bc0b853127e127ba3" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.37" + "version": "==1.34.52" }, "botocore": { "hashes": [ - "sha256:2a5bf33aacd2d970afd3d492e179e06ea98a5469030d5cfe7a2ad9995f7bb2ef", - "sha256:3c46ddb1679e6ef45ca78b48665398636bda532a07cd476e4b500697d13d9a99" + "sha256:05567d8aba344826060481ea309555432c96f0febe22bee7cf5a3b6d3a03cec8", + "sha256:187da93aec3f2e87d8a31eced16fa2cb9c71fe2d69b0a797f9f7a9220f5bf7ae" ], "markers": "python_version >= '3.8'", - "version": "==1.34.37" + "version": "==1.34.52" }, "botocore-stubs": { "hashes": [ - "sha256:087cd42973edcb5527dc97eec87fa29fffecc39691249486e02045677d4a2dbe", - "sha256:d6bcea8a6872aa46d389027dc5c022241fd0a2047a8b858aa5005e6151ed30a7" + "sha256:8748b9fe01f66bb1e7b13f45e3336e2e2c5460d232816d45941573425459c66e", + "sha256:d0f4d9859d9f6affbe4b0b46e37fe729860eaab55ebefe7e09cf567396b2feda" ], "markers": "python_version >= '3.8' and python_version < '4.0'", - "version": "==1.34.37" + "version": "==1.34.51" }, "click": { "hashes": [ @@ -1492,11 +1503,11 @@ }, "rich": { "hashes": [ - "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa", - "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235" + "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222", + "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432" ], "markers": "python_full_version >= '3.7.0'", - "version": "==13.7.0" + "version": "==13.7.1" }, "s3transfer": { "hashes": [ @@ -1532,11 +1543,11 @@ }, "stevedore": { "hashes": [ - "sha256:8cc040628f3cea5d7128f2e76cf486b2251a4e543c7b938f58d9a377f6694a2d", - "sha256:a54534acf9b89bc7ed264807013b505bf07f74dbe4bcfa37d32bd063870b087c" + "sha256:1c15d95766ca0569cad14cb6272d4d31dae66b011a929d7c18219c176ea1b5c9", + "sha256:46b93ca40e1114cea93d738a6c1e365396981bb6bb78c27045b7587c9473544d" ], "markers": "python_version >= '3.8'", - "version": "==5.1.0" + "version": "==5.2.0" }, "tomli": { "hashes": [ @@ -1548,11 +1559,11 @@ }, "types-awscrt": { "hashes": [ - "sha256:06a859189a329ca8e66d56ceeef2391488e39b878fbd2141f115eab4d416fe22", - "sha256:f61a120d3e98ee1387bc5ca4b93437f258cc5c2af1f55f8634ec4cee5729f178" + "sha256:10245570c7285e949362b4ae710c54bf285d64a27453d42762477bcee5cd77a3", + "sha256:73be0a2720d6f76b924df6917d4edf4c9958f83e5c25bf7d9f0c1e9cdf836941" ], "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==0.20.3" + "version": "==0.20.4" }, "types-cachetools": { "hashes": [ @@ -1580,12 +1591,12 @@ }, "types-requests": { "hashes": [ - "sha256:03a28ce1d7cd54199148e043b2079cdded22d6795d19a2c2a6791a4b2b5e2eb5", - "sha256:9592a9a4cb92d6d75d9b491a41477272b710e021011a2a3061157e2fb1f1a5d1" + "sha256:a82807ec6ddce8f00fe0e949da6d6bc1fbf1715420218a9640d695f70a9e5a9b", + "sha256:f1721dba8385958f504a5386240b92de4734e047a08a40751c1654d1ac3349c5" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.31.0.20240125" + "version": "==2.31.0.20240218" }, "types-s3transfer": { "hashes": [ @@ -1597,12 +1608,12 @@ }, "typing-extensions": { "hashes": [ - "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", - "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", + "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.9.0" + "version": "==4.10.0" }, "urllib3": { "hashes": [ diff --git a/src/docker-compose.yml b/src/docker-compose.yml index fdf069f56..600347fa8 100644 --- a/src/docker-compose.yml +++ b/src/docker-compose.yml @@ -58,6 +58,8 @@ services: - AWS_S3_SECRET_ACCESS_KEY - AWS_S3_REGION - AWS_S3_BUCKET_NAME + # File encryption credentials + - SECRET_ENCRYPT_METADATA stdin_open: true tty: true ports: diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 009baa1c6..f763a71ce 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -74,6 +74,8 @@ secret_aws_s3_key_id = secret("access_key_id", None) or secret("AWS_S3_ACCESS_KE secret_aws_s3_key = secret("secret_access_key", None) or secret("AWS_S3_SECRET_ACCESS_KEY", None) secret_aws_s3_bucket_name = secret("bucket", None) or secret("AWS_S3_BUCKET_NAME", None) +secret_encrypt_metadata = secret("SECRET_ENCRYPT_METADATA", None) + secret_registry_cl_id = secret("REGISTRY_CL_ID") secret_registry_password = secret("REGISTRY_PASSWORD") secret_registry_cert = b64decode(secret("REGISTRY_CERT", "")) @@ -94,6 +96,7 @@ DEBUG = env_debug # Controls production specific feature toggles IS_PRODUCTION = env_is_production +SECRET_ENCRYPT_METADATA = secret_encrypt_metadata # Applications are modular pieces of code. # They are provided by Django, by third-parties, or by yourself. diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 69d111f4c..a27199cdb 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -5,6 +5,7 @@ import os import pyzipper from django.core.management import BaseCommand +from django.conf import settings from registrar.utility import csv_export from registrar.utility.s3_bucket import S3ClientHelper from ...utility.email import send_templated_email, EmailSendingError @@ -61,27 +62,28 @@ class Command(BaseCommand): We want to make sure to upload to s3 for back up And now we also want to get the file and encrypt it so we can send it in an email """ - unencrypted_metadata_input = s3_client.get_file(file_name) - # Encrypt metadata into a zip file # pre-setting zip file name encrypted_metadata_output = 'encrypted_metadata.zip' - # set this to be an env var somewhere - password = b'somepasswordhere' + + # Secret is encrypted into getgov-credentials + # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when ready + # encrypted_metadata is the encrypted output - encrypted_metadata = _encrypt_metadata(unencrypted_metadata_input, encrypted_metadata_output, password) + encrypted_metadata = self._encrypt_metadata(s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA)) print("encrypted_metadata is:", encrypted_metadata) # Send the metadata file that is zipped # Q: Would we set the vars I set in email.py here to pass in to the helper function or best way to invoke # send_templated_email(encrypted_metadata, attachment=True) - def _encrypt_metadata(input_file, output_file, password): - with open(input_file, 'rb') as f_in: - with pyzipper.AESZipFile(output_file, 'w', compression=pyzipper.ZIP_LZMA, encryption=pyzipper.WZ_AES) as f_out: - f_out.setpassword(password) - f_out.writestr(input_file, f_in.read()) - return output_file + def _encrypt_metadata(self, input_file, output_file, password): + # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities + # Could also use compression=pyzipper.ZIP_LZMA? + with pyzipper.AESZipFile(output_file, 'w', compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES) as f_out: + f_out.setpassword(password) + f_out.writestr('encrypted_metadata.txt', input_file) + return output_file diff --git a/src/requirements.txt b/src/requirements.txt index a6130a3bf..4b904cddd 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,14 +1,14 @@ -i https://pypi.python.org/simple annotated-types==0.6.0; python_version >= '3.8' asgiref==3.7.2; python_version >= '3.7' -boto3==1.34.37; python_version >= '3.8' -botocore==1.34.37; python_version >= '3.8' -cachetools==5.3.2; python_version >= '3.7' +boto3==1.34.52; python_version >= '3.8' +botocore==1.34.52; python_version >= '3.8' +cachetools==5.3.3; python_version >= '3.7' certifi==2024.2.2; python_version >= '3.6' cfenv==0.5.3 cffi==1.16.0; platform_python_implementation != 'PyPy' charset-normalizer==3.3.2; python_full_version >= '3.7.0' -cryptography==42.0.2; python_version >= '3.7' +cryptography==42.0.5; python_version >= '3.7' defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' dj-database-url==2.1.0 dj-email-url==1.0.6 @@ -23,12 +23,12 @@ django-login-required-middleware==0.9.0 django-phonenumber-field[phonenumberslite]==7.3.0; python_version >= '3.8' django-widget-tweaks==1.5.0; python_version >= '3.8' environs[django]==10.3.0; python_version >= '3.8' -faker==23.1.0; python_version >= '3.8' +faker==23.3.0; python_version >= '3.8' fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a3a9a345f0ef561c furl==2.1.3 -future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -gevent==23.9.1; python_version >= '3.8' -geventconnpool@ git+https://github.com/rasky/geventconnpool.git@1bbb93a714a331a069adf27265fe582d9ba7ecd4 +future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +gevent==24.2.1; python_version >= '3.8' +geventconnpool@ git+https://github.com/rasky/geventconnpool.git greenlet==3.0.3; python_version >= '3.7' gunicorn==21.2.0; python_version >= '3.5' idna==3.6; python_version >= '3.5' @@ -36,27 +36,28 @@ jmespath==1.0.1; python_version >= '3.7' lxml==5.1.0; python_version >= '3.6' mako==1.3.2; python_version >= '3.8' markupsafe==2.1.5; python_version >= '3.7' -marshmallow==3.20.2; python_version >= '3.8' +marshmallow==3.21.0; python_version >= '3.8' oic==1.6.1; python_version ~= '3.7' orderedmultidict==1.0.1 packaging==23.2; python_version >= '3.7' -phonenumberslite==8.13.29 +phonenumberslite==8.13.31 psycopg2-binary==2.9.9; python_version >= '3.7' pycparser==2.21 pycryptodomex==3.20.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pydantic==2.6.1; python_version >= '3.8' -pydantic-core==2.16.2; python_version >= '3.8' -pydantic-settings==2.1.0; python_version >= '3.8' +pydantic==2.6.3; python_version >= '3.8' +pydantic-core==2.16.3; python_version >= '3.8' +pydantic-settings==2.2.1; python_version >= '3.8' pyjwkest==1.4.2 python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-dotenv==1.0.1; python_version >= '3.8' +pyzipper==0.3.6; python_version >= '3.4' requests==2.31.0; python_version >= '3.7' s3transfer==0.10.0; python_version >= '3.8' -setuptools==69.0.3; python_version >= '3.8' +setuptools==69.1.1; python_version >= '3.8' six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sqlparse==0.4.4; python_version >= '3.5' -typing-extensions==4.9.0; python_version >= '3.8' +typing-extensions==4.10.0; python_version >= '3.8' urllib3==2.0.7; python_version >= '3.7' whitenoise==6.6.0; python_version >= '3.8' zope.event==5.0; python_version >= '3.7' -zope.interface==6.1; python_version >= '3.7' +zope.interface==6.2; python_version >= '3.7' From 7d65b2cf5cfea3a816e5fc44902f67a7d5993a4d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:15:21 -0700 Subject: [PATCH 011/125] JS changes --- src/registrar/assets/js/get-gov-admin.js | 66 ++++++++- .../admin/domain_application_change_form.html | 135 ++++++++++-------- 2 files changed, 135 insertions(+), 66 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 096f7a626..47351a608 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -35,14 +35,20 @@ function openInNewTab(el, removeAttribute = false){ let input = document.createElement("input"); input.type = "submit"; - input.name = button.name; - input.value = button.value; + + if(button.name){ + input.name = button.name; + } + + if(button.value){ + input.value = button.value; + } + input.style.display = "none" // Add the hidden input to the form form.appendChild(input); button.addEventListener("click", () => { - console.log("clicking") input.click(); }) }) @@ -50,6 +56,60 @@ function openInNewTab(el, removeAttribute = false){ createPhantomModalFormButtons(); })(); + +/** An IIFE for DomainApplication to hook a modal to a dropdown option. + * This intentionally does not interact with createPhantomModalFormButtons() +*/ +(function (){ + function displayModalOnDropdownClick(){ + // Grab the invisible element that will hook to the modal. + // This doesn't technically need to be done with one, but this is simpler to manage. + let linkClickedDisplaysModal = document.getElementById("invisible-ineligible-modal-toggler") + let statusDropdown = document.getElementById("id_status") + + // If these exist all at the same time, we're on the right page + if (linkClickedDisplaysModal && statusDropdown && statusDropdown.value){ + // Store the previous value in the event the user cancels. + // We only need to do this if we're on the correct page. + let previousValue = statusDropdown.value; + // Because the modal button does not have the class "dja-form-placeholder", + // it will not be affected by the createPhantomModalFormButtons() function. + let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); + if (cancelButton){ + console.log(`This is the previous val: ${previousValue}`) + cancelButton.addEventListener('click', function() { + // Revert the dropdown to its previous value + statusDropdown.value = previousValue; + }); + + // Add a change event listener to the dropdown. + statusDropdown.addEventListener('change', function() { + // Check if "Ineligible" is selected + if (this.value && this.value.toLowerCase() === "ineligible") { + // Display the modal. + linkClickedDisplaysModal.click() + } + + // Update previousValue if another option is selected and confirmed + previousValue = this.value; + console.log(`This is the previous val NOW: ${previousValue}`) + }); + + } else{ + console.error("displayModalOnDropdownClick() -> No cancel button defined.") + } + + } + } + + // Adds event listeners on the confirm and cancel modal buttons + function handleModalButtons(){ + + } + + displayModalOnDropdownClick(); +})(); + /** An IIFE for pages in DjangoAdmin which may need custom JS implementation. * Currently only appends target="_blank" to the domain_form object, * but this can be expanded. diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index 26abbec18..f6380fb82 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -1,6 +1,12 @@ {% extends 'admin/change_form.html' %} {% load i18n static %} +{% block field_sets %} + {# Create an invisible tag so that we can use a click event to toggle the modal. #} + + {{ block.super }} +{% endblock %} + {% block submit_buttons_bottom %} {% comment %} Modals behave very weirdly in django admin. @@ -16,72 +22,75 @@ In addition, the modal element MUST be placed low in the DOM. The script loads slower on DJA than on other portions of the application, so this means that it will briefly "populate", causing unintended visual effects. {% endcomment %} -{# Create a modal for when a domain is marked as ineligible #} -
-
-
- -
-

- When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: -

-
    -
  • They cannot edit the ineligible request or any other pending requests.
  • -
  • They cannot manage any of their approved domains.
  • -
  • They cannot initiate a new domain request.
  • -
-

- This action can be reversed, if needed. -

-

- {# Acts as a
#} -

- Domain: {{ original.name }} - New status: {{ original.State }} -

-
+ {# Create a modal for when a domain is marked as ineligible #} +
+
+
+ +
+

+ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows: +

+
    +
  • They cannot edit the ineligible request or any other pending requests.
  • +
  • They cannot manage any of their approved domains.
  • +
  • They cannot initiate a new domain request.
  • +
+

+ The restrictions will not take effect until you “save” the changes for this domain request. + This action can be reversed, if needed. +

+

+ {# Acts as a
#} +

+ Domain: {{ original.name }} + New status: {{ original.State }} +

+
- +
- -
{{ block.super }} {% endblock %} \ No newline at end of file From 7a7000cf9d358930280585cdb74c8f366ba815bd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:31:08 -0700 Subject: [PATCH 012/125] Fix bug --- src/registrar/assets/js/get-gov-admin.js | 6 ------ .../django/admin/domain_application_change_form.html | 6 +++--- .../templates/django/admin/domain_change_form.html | 6 +++--- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 47351a608..ebcca16d7 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -91,7 +91,6 @@ function openInNewTab(el, removeAttribute = false){ } // Update previousValue if another option is selected and confirmed - previousValue = this.value; console.log(`This is the previous val NOW: ${previousValue}`) }); @@ -102,11 +101,6 @@ function openInNewTab(el, removeAttribute = false){ } } - // Adds event listeners on the confirm and cancel modal buttons - function handleModalButtons(){ - - } - displayModalOnDropdownClick(); })(); diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index f6380fb82..6e6ab3723 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -39,9 +39,9 @@ When a domain request is in ineligible status, the registrant's permissions within the registrar are restricted as follows:

    -
  • They cannot edit the ineligible request or any other pending requests.
  • -
  • They cannot manage any of their approved domains.
  • -
  • They cannot initiate a new domain request.
  • +
  • They cannot edit the ineligible request or any other pending requests.
  • +
  • They cannot manage any of their approved domains.
  • +
  • They cannot initiate a new domain request.

The restrictions will not take effect until you “save” the changes for this domain request. diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 61007c3d4..393983e32 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -133,9 +133,9 @@ When a domain is on hold:

    -
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • -
  • The domain will still appear in the registrar / admin.
  • -
  • Domain managers won’t be able to edit the domain.
  • +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain will still appear in the registrar / admin.
  • +
  • Domain managers won’t be able to edit the domain.

This action can be reversed, if needed. From 0a174e5d29b9625c0d2454475b67dc8519656541 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 13:58:33 -0500 Subject: [PATCH 013/125] Reports, chart wip --- src/registrar/admin.py | 143 ++++++----- src/registrar/assets/js/get-gov-admin.js | 26 +- src/registrar/assets/sass/_theme/_admin.scss | 17 ++ src/registrar/config/settings.py | 2 +- src/registrar/config/urls.py | 42 +++- src/registrar/signals.py | 2 + src/registrar/templates/admin/analytics.html | 105 ++++++-- src/registrar/templates/admin/base_site.html | 2 + src/registrar/tests/test_admin_views.py | 2 +- src/registrar/tests/test_reports.py | 2 +- src/registrar/utility/csv_export.py | 237 +++++++++++++++++-- src/registrar/views/admin_views.py | 58 ++++- 12 files changed, 498 insertions(+), 140 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 34270584a..9bc77b029 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -27,6 +27,7 @@ from django_fsm import TransitionNotAllowed # type: ignore from django.utils.safestring import mark_safe from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from django.utils.translation import gettext_lazy as _ @@ -363,6 +364,75 @@ class UserContactInline(admin.StackedInline): model = models.Contact +def user_analytics(request): + + end_date = datetime.datetime.today() + start_date = datetime.datetime.today() - datetime.timedelta(days=30) + + last_30_days_applications = models.DomainApplication.objects.filter( + created_at__gt=start_date + ) + avg_approval_time = last_30_days_applications.annotate( + approval_time=F("approved_domain__created_at") - F("created_at") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # format the timedelta? + avg_approval_time = str(avg_approval_time) + + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] + + logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # get number of ready domains, counts by org type and election office + # add to context + + # get number of submitted request counts by org type and election office + # add to context + + context = dict( + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + applications_last_30_days=last_30_days_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ), + ) + return render(request, "admin/analytics.html", context) + class MyUserAdmin(BaseUserAdmin): """Custom user admin class to use our inlines.""" @@ -464,79 +534,6 @@ class MyUserAdmin(BaseUserAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name", "email"] - def get_urls(self): - """Map a new page in admin for analytics.""" - urlpatterns = super().get_urls() - - # Used to extrapolate a path name, for instance - # name="{app_label}_{model_name}_export_data_type" - domain_path_meta = self.model._meta.app_label, models.Domain._meta.model_name - - my_urls = [ - path( - "analytics/", - self.admin_site.admin_view(self.user_analytics), - name="user_analytics", - ), - path( - "export_data_type/", - self.export_data_type, - name="%s_%s_export_data_type" % domain_path_meta, - ), - path( - "export_data_full/", - self.export_data_full, - name="%s_%s_export_data_full" % domain_path_meta, - ), - path( - "export_data_federal/", - self.export_data_federal, - name="%s_%s_export_data_federal" % domain_path_meta, - ), - ] - - return my_urls + urlpatterns - - def export_data_type(self, request): - # match the CSV example with all the fields - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) - return response - - def export_data_full(self, request): - # Smaller export based on 1 - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) - return response - - def export_data_federal(self, request): - # Federal only - response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) - return response - - def user_analytics(self, request): - last_30_days_applications = models.DomainApplication.objects.filter( - created_at__gt=datetime.datetime.today() - datetime.timedelta(days=30) - ) - avg_approval_time = last_30_days_applications.annotate( - approval_time=F("approved_domain__created_at") - F("created_at") - ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) - context = dict( - **self.admin_site.each_context(request), - data=dict( - user_count=models.User.objects.all().count(), - domain_count=models.Domain.objects.all().count(), - applications_last_30_days=last_30_days_applications.count(), - average_application_approval_time_last_30_days=avg_approval_time, - ), - ) - return render(request, "admin/analytics.html", context) def get_search_results(self, request, queryset, search_term): """ Override for get_search_results. This affects any upstream model using autocomplete_fields, diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ff73acb65..618cc284c 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -322,23 +322,25 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, // Default the value of the end date input field to the current date let endDateInput =document.getElementById('end'); - let exportGrowthReportButton = document.getElementById('exportLink'); + let exportButtons = document.querySelectorAll('.exportLink'); - if (exportGrowthReportButton) { + if (exportButtons.length > 0) { startDateInput.value = currentDate; endDateInput.value = currentDate; - exportGrowthReportButton.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = document.getElementById('exportLink').dataset.exportUrl; + exportButtons.forEach((btn) => { + btn.addEventListener('click', function() { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); }); } diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 0d232ff41..c74daf678 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -303,3 +303,20 @@ input.admin-confirm-button { display: contents !important; } } + +.usa-button-group { + margin-left: -0.25rem!important; + padding-left: 0!important; + .usa-button-group__item { + list-style-type: none; + line-height: normal; + } + .button { + display: inline-block; + padding: 10px 8px; + line-height: normal; + } + .usa-icon { + top: 2px; + } +} diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index bb8e22ad7..56f3c2090 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -330,7 +330,7 @@ CSP_FORM_ACTION = allowed_sources # Google analytics requires that we relax our otherwise # strict CSP by allowing scripts to run from their domain # and inline with a nonce, as well as allowing connections back to their domain -CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] +CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/", "https://cdn.jsdelivr.net/npm/chart.js"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 4bd7b4baf..a9fee650e 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,9 +9,8 @@ from django.urls import include, path from django.views.generic import RedirectView from registrar import views - -from registrar.views.admin_views import ExportData - +from registrar.admin import user_analytics +from registrar.views.admin_views import ExportDataDomainGrowth, ExportDataFederal, ExportDataFull, ExportDataManagedVsUnmanaged, ExportDataRequests, ExportDataType from registrar.views.application import Step from registrar.views.utility import always_404 @@ -52,7 +51,42 @@ urlpatterns = [ "admin/logout/", RedirectView.as_view(pattern_name="logout", permanent=False), ), - path("export_data/", ExportData.as_view(), name="admin_export_data"), + path( + "admin/analytics/export_data_type/", + ExportDataType.as_view(), + name="export_data_type", + ), + path( + "admin/analytics/export_data_full/", + ExportDataFull.as_view(), + name="export_data_full", + ), + path( + "admin/analytics/export_data_federal/", + ExportDataFederal.as_view(), + name="export_data_federal", + ), + path( + "admin/analytics/export_domain_growth/", + ExportDataDomainGrowth.as_view(), + name="export_domain_growth", + ), + path( + "admin/analytics/export_managed_unmanaged/", + ExportDataManagedVsUnmanaged.as_view(), + name="export_managed_unmanaged", + ), + path( + "admin/analytics/export_requests/", + ExportDataRequests.as_view(), + name="export_requests", + ), + path( + "admin/analytics/", + admin.site.admin_view(user_analytics), + name="user_analytics", + ), + path("admin/", admin.site.urls), path( "application//edit/", diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 4e7768ef4..ef09e605b 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,6 +27,7 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") + logger.info(f'in handle_profile first {instance}') is_new_user = kwargs.get("created", False) @@ -36,6 +37,7 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact + logger.info(f'inside no matching contacts for first {first_name} last {last_name} email {email}') Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 82081d629..f65aa77cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,25 +1,11 @@ {% extends "admin/base_site.html" %} - +{% load static %} {% block content_title %}

Registrar Analytics

{% endblock %} {% block content %} - {% block object-tools %} - - {% endblock %} -

At a glance

@@ -34,17 +20,46 @@
-

Domain growth

+

Current domains

+ +
+ +
+

Growth reports

{% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_data/ + The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ See the commit "Review for ticket #999" {% endcomment %} -
-
+
+
@@ -52,8 +67,58 @@
-
+ +
    +
  • + +
  • +
  • + +
  • +
  • + +
  • +
  • + +
  • +
+ +
+
+ +
+
+ +
+
+
diff --git a/src/registrar/templates/admin/base_site.html b/src/registrar/templates/admin/base_site.html index 73e9ba1f0..58843421a 100644 --- a/src/registrar/templates/admin/base_site.html +++ b/src/registrar/templates/admin/base_site.html @@ -20,7 +20,9 @@ > + + {% endblock %} {% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index aa150d55c..e55175db9 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -26,7 +26,7 @@ class TestViews(TestCase): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 011c60b93..c00c2b221 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -542,7 +542,7 @@ class ExportDataTest(MockEppLib): are pulled when the growth report conditions are applied to export_domains_to_writed. Test that ready domains are sorted by first_ready/deleted dates first, names second. - We considered testing export_data_growth_to_csv which calls write_body + We considered testing export_data_domain_growth_to_csv which calls write_body and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 90e80f551..1764536b5 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,6 +2,7 @@ import csv import logging from datetime import datetime from registrar.models.domain import Domain +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from django.utils import timezone from django.core.paginator import Paginator @@ -19,10 +20,8 @@ def write_header(writer, columns): Receives params from the parent methods and outputs a CSV with a header row. Works with write_header as long as the same writer object is passed. """ - writer.writerow(columns) - def get_domain_infos(filter_condition, sort_fields): domain_infos = ( DomainInformation.objects.select_related("domain", "authorizing_official") @@ -43,7 +42,6 @@ def get_domain_infos(filter_condition, sort_fields): ) return domain_infos_cleaned - def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" @@ -104,7 +102,6 @@ def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None row = [FIELDS.get(column, "") for column in columns] return row - def _get_security_emails(sec_contact_ids): """ Retrieve security contact emails for the given security contact IDs. @@ -126,7 +123,6 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict - def update_columns_with_domain_managers(columns, max_dm_count): """ Update the columns list to include "Domain manager email {#}" headers @@ -135,7 +131,6 @@ def update_columns_with_domain_managers(columns, max_dm_count): for i in range(1, max_dm_count + 1): columns.append(f"Domain manager email {i}") - def write_csv( writer, columns, @@ -148,7 +143,7 @@ def write_csv( Receives params from the parent methods and outputs a CSV with fltered and sorted domains. Works with write_header as longas the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_growth_to_csv calls write_body twice + should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ all_domain_infos = get_domain_infos(filter_condition, sort_fields) @@ -158,15 +153,15 @@ def write_csv( security_emails_dict = _get_security_emails(sec_contact_ids) - # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_domain_infos, 1000) - if get_domain_managers and len(all_domain_infos) > 0: # We want to get the max amont of domain managers an # account has to set the column header dynamically max_dm_count = max(len(domain_info.domain.permissions.all()) for domain_info in all_domain_infos) update_columns_with_domain_managers(columns, max_dm_count) + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_domain_infos, 1000) + for page_num in paginator.page_range: page = paginator.page(page_num) rows = [] @@ -185,6 +180,82 @@ def write_csv( writer.writerows(rows) +def get_domain_requests(filter_condition, sort_fields): + domain_requests = ( + DomainApplication.objects.all() + .filter(**filter_condition) + .order_by(*sort_fields) + ) + + return domain_requests + +def parse_request_row(columns, request: DomainApplication): + """Given a set of columns, generate a new row from cleaned column data""" + + requested_domain_name = 'No requested domain' + + # Domain should never be none when parsing this information + if request.requested_domain is not None: + domain = request.requested_domain + requested_domain_name = domain.name + + domain = request.requested_domain # type: ignore + + if request.federal_type: + request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" + else: + request_type = request.get_organization_type_display() + + # create a dictionary of fields which can be included in output + FIELDS = { + "Requested domain": requested_domain_name, + "Status": request.get_status_display(), + "Organization type": request_type, + "Agency": request.federal_agency, + "Organization name": request.organization_name, + "City": request.city, + "State": request.state_territory, + "AO email": request.authorizing_official.email if request.authorizing_official else " ", + "Security contact email": request, + "Created at": request.created_at, + "Submission date": request.submission_date, + } + + row = [FIELDS.get(column, "") for column in columns] + return row + +def write_requests_csv( + writer, + columns, + sort_fields, + filter_condition, + should_write_header=True, +): + """ + """ + + all_requetsts = get_domain_requests(filter_condition, sort_fields) + + # Reduce the memory overhead when performing the write operation + paginator = Paginator(all_requetsts, 1000) + + for page_num in paginator.page_range: + page = paginator.page(page_num) + rows = [] + for request in page.object_list: + try: + row = parse_request_row(columns, request) + rows.append(row) + except ValueError: + # This should not happen. If it does, just skip this row. + # It indicates that DomainInformation.domain is None. + logger.error("csv_export -> Error when parsing row, domain was None") + continue + + if should_write_header: + write_header(writer, columns) + + writer.writerows(rows) def export_data_type_to_csv(csv_file): """All domains report with extra columns""" @@ -222,7 +293,6 @@ def export_data_type_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) - def export_data_full_to_csv(csv_file): """All domains report""" @@ -253,7 +323,6 @@ def export_data_full_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def export_data_federal_to_csv(csv_file): """Federal domains report""" @@ -285,18 +354,21 @@ def export_data_federal_to_csv(csv_file): } write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) - def get_default_end_date(): # Default to now() return timezone.now() +def format_start_date(start_date): + return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() -def export_data_growth_to_csv(csv_file, start_date, end_date): +def format_end_date(end_date): + return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + +def export_data_domain_growth_to_csv(csv_file, start_date, end_date): """ Growth report: Receive start and end dates from the view, parse them. @@ -305,16 +377,9 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): the start and end dates. Specify sort params for both lists. """ - start_date_formatted = ( - timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() - ) - - end_date_formatted = ( - timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - ) - + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) writer = csv.writer(csv_file) - # define columns to include in export columns = [ "Domain name", @@ -359,3 +424,127 @@ def export_data_growth_to_csv(csv_file, start_date, end_date): get_domain_managers=False, should_write_header=False, ) + +def get_sliced_domains(filter_condition): + """ + """ + + domains = DomainInformation.objects.all().filter(**filter_condition) + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domains.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + +def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + writer.writerow(["START DATE"]) + writer.writerow([]) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__created_at__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_start_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + writer.writerow(["END DATE"]) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_managed_domains_end_date, get_domain_managers=True, should_write_header=True) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS"]) + writer.writerow(["FEDERAL", "INTERSTATE", "STATE_OR_TERRITORY", "TRIBAL", "COUNTY", "CITY", "SPECIAL_DISTRICT", "SCHOOL_DISTRICT", "ELECTION OFFICE"]) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_csv(writer, columns, sort_fields, filter_unmanaged_domains_end_date, get_domain_managers=True, should_write_header=True) + +def export_data_requests_to_csv(csv_file, start_date, end_date): + """ + """ + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + # "domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + "submission_date__gte": start_date_formatted, + } + + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f7164663b..4d93aa54b 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -8,9 +8,32 @@ from registrar.utility import csv_export import logging logger = logging.getLogger(__name__) + +class ExportDataType(View): + def get(self, request, *args, **kwargs): + # match the CSV example with all the fields + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' + csv_export.export_data_type_to_csv(response) + return response + +class ExportDataFull(View): + def get(self, request, *args, **kwargs): + # Smaller export based on 1 + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' + csv_export.export_data_full_to_csv(response) + return response + +class ExportDataFederal(View): + def get(self, request, *args, **kwargs): + # Federal only + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' + csv_export.export_data_federal_to_csv(response) + return response - -class ExportData(View): +class ExportDataDomainGrowth(View): def get(self, request, *args, **kwargs): # Get start_date and end_date from the request's GET parameters # #999: not needed if we switch to django forms @@ -19,8 +42,35 @@ class ExportData(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use # in context to display this data in the template. - csv_export.export_data_growth_to_csv(response, start_date, end_date) + csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) return response + +class ExportDataManagedVsUnmanaged(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + csv_export.export_data_managed_vs_unamanaged_domains(response, start_date, end_date) + + return response + +class ExportDataRequests(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' + # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use + # in context to display this data in the template. + csv_export.export_data_requests_to_csv(response, start_date, end_date) + + return response \ No newline at end of file From 0ef72ad016a975f73922286723e0c5f0ebd3bd29 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:05:19 -0700 Subject: [PATCH 014/125] Add custom desc on delete button --- src/registrar/admin.py | 12 +++++++++++ src/registrar/assets/js/get-gov-admin.js | 1 - .../admin/domain_application_change_form.html | 4 ++-- .../django/admin/domain_change_form.html | 4 ++-- .../admin/domain_delete_confirmation.html | 21 +++++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 src/registrar/templates/django/admin/domain_delete_confirmation.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 942ae6162..fd6ce45a7 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1339,6 +1339,18 @@ class DomainAdmin(ListHeaderAdmin): # Table ordering ordering = ["name"] + def delete_view(self, request, object_id, extra_context=None): + """ + Custom delete_view to perform additional actions or customize the template. + """ + + # Set the delete template to a custom one + self.delete_confirmation_template = "django/admin/domain_delete_confirmation.html" + + response = super().delete_view(request, object_id, extra_context=extra_context) + + return response + def changeform_view(self, request, object_id=None, form_url="", extra_context=None): """Custom changeform implementation to pass in context information""" if extra_context is None: diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index ebcca16d7..8ecf2cbee 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -76,7 +76,6 @@ function openInNewTab(el, removeAttribute = false){ // it will not be affected by the createPhantomModalFormButtons() function. let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); if (cancelButton){ - console.log(`This is the previous val: ${previousValue}`) cancelButton.addEventListener('click', function() { // Revert the dropdown to its previous value statusDropdown.value = previousValue; diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index 6e6ab3723..f0e4cfe4f 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -48,10 +48,10 @@ This action can be reversed, if needed.

+ Domain: {{ original.requested_domain.name }} {# Acts as a
#}

- Domain: {{ original.name }} - New status: {{ original.State }} + New status: {{ original.ApplicationStatus.INELIGIBLE|capfirst }}

diff --git a/src/registrar/templates/django/admin/domain_change_form.html b/src/registrar/templates/django/admin/domain_change_form.html index 393983e32..818522c8d 100644 --- a/src/registrar/templates/django/admin/domain_change_form.html +++ b/src/registrar/templates/django/admin/domain_change_form.html @@ -144,7 +144,7 @@ Domain: {{ original.name }} {# Acts as a
#}
- New status: {{ original.State.ON_HOLD }} + New status: {{ original.State.ON_HOLD|capfirst }}

@@ -211,7 +211,7 @@ Domain: {{ original.name }} {# Acts as a
#}
- New status: {{ original.State.DELETED }} + New status: {{ original.State.DELETED|capfirst }}

diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html new file mode 100644 index 000000000..793a28c4c --- /dev/null +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -0,0 +1,21 @@ +{% extends 'admin/delete_confirmation.html' %} +{% load i18n static %} + +{% block content %} +{# TODO modify the "Are you sure?" to the text content below.. #} +{% comment %} +

Are you sure you want to remove this domain from the registry?

+{% endcomment %} +

Description

+

When a domain is removed from the registry:

+ +
    +
  • The domain and its subdomains won’t resolve in DNS. Any infrastructure (like websites) will go offline.
  • +
  • The domain will still appear in the registrar / admin.
  • +
  • Domain managers won’t be able to edit the domain.
  • +
+ +

This action cannot be undone.

+ +{{ block.super }} +{% endblock %} From da47cc6f7f661abba3a9dee51dc798ada775b22a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 29 Feb 2024 18:46:45 -0500 Subject: [PATCH 015/125] Add charts to dashboard --- src/registrar/admin.py | 75 ++++++++++++++++---- src/registrar/templates/admin/analytics.html | 53 +++++++++++++- src/registrar/utility/csv_export.py | 27 ++++++- 3 files changed, 140 insertions(+), 15 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 9bc77b029..ca6b9bc87 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -384,39 +384,78 @@ def user_analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) + # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_start_date = [10, 20, 50, 0, 0, 12, 6, 5] - - logger.info(f"managed_domains_sliced_at_start_date {managed_domains_sliced_at_start_date}") - + filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_start_date = [15, 13, 60, 0, 2, 11, 6, 5] filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) - managed_domains_sliced_at_end_date = [12, 20, 60, 0, 0, 12, 6, 4] filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - unmanaged_domains_sliced_at_end_date = [5, 40, 55, 0, 0, 12, 6, 5] + + # Ready and Deleted domains + filter_ready_domains_start_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": start_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_end_date = { + "domain__state__in": [Domain.State.DELETED], + "domain__first_ready__lte": end_date_formatted, + } + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + - # get number of ready domains, counts by org type and election office - # add to context + # Created and Submitted requests + filter_requests_start_date = { + "submission_date__lte": start_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + + filter_submitted_requests_start_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - # get number of submitted request counts by org type and election office - # add to context + filter_requests_end_date = { + "submission_date__lte": end_date_formatted, + } + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_end_date = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( **admin.site.each_context(request), @@ -425,10 +464,22 @@ def user_analytics(request): domain_count=models.Domain.objects.all().count(), applications_last_30_days=last_30_days_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_at_end_date=submitted_requests_at_end_date, + ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index f65aa77cf..735386d38 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -7,7 +7,10 @@ {% block content %}
-
+ +
+
+

At a glance

    @@ -19,7 +22,10 @@
-
+
+
+ +

Current domains

    @@ -48,6 +54,9 @@
+
+
+

Growth reports

@@ -119,8 +128,48 @@
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
{% endblock %} diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1764536b5..ce19182b2 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -450,6 +450,31 @@ def get_sliced_domains(filter_condition): school_district, election_board] +def get_sliced_requests(filter_condition): + """ + """ + + domain_requests = DomainApplication.objects.all().filter(**filter_condition) + federal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + interstate = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + state_or_territory = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).count() + tribal = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() + county = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() + city = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() + special_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() + school_district = domain_requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() + election_board = domain_requests.filter(is_election_board=True).count() + + return [federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board] + def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): """ """ @@ -470,7 +495,7 @@ def export_data_managed_vs_unamanaged_domains(csv_file, start_date, end_date): filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__created_at__lte": start_date_formatted, + "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) From 74f203448771cfbda77afce2279bd9f9b704a195 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 29 Feb 2024 22:17:48 -0800 Subject: [PATCH 016/125] Email business logic --- .../runbooks/update_python_dependencies.md | 6 +- .../generate_current_metadata_report.py | 20 ++-- src/registrar/utility/email.py | 110 ++++++++---------- 3 files changed, 64 insertions(+), 72 deletions(-) diff --git a/docs/operations/runbooks/update_python_dependencies.md b/docs/operations/runbooks/update_python_dependencies.md index 04fb936c6..468270d09 100644 --- a/docs/operations/runbooks/update_python_dependencies.md +++ b/docs/operations/runbooks/update_python_dependencies.md @@ -3,7 +3,7 @@ 1. Check the [Pipfile](../../../src/Pipfile) for pinned dependencies and manually adjust the version numbers 2. Run `docker-compose stop` to spin down the current containers and images so we can start afresh -2. Run +3. Run cd src docker-compose run app bash -c "pipenv lock && pipenv requirements > requirements.txt" @@ -13,9 +13,9 @@ It is necessary to use `bash -c` because `run pipenv requirements` will not recognize that it is running non-interactively and will include garbage formatting characters. The requirements.txt is used by Cloud.gov. It is needed to work around a bug in the CloudFoundry buildpack version of Pipenv that breaks on installing from a git repository. -3. Change geventconnpool back to what it was originally within the Pipfile.lock and requirements.txt. +4. Change geventconnpool back to what it was originally within the Pipfile.lock and requirements.txt. This is done by either saving what it was originally or opening a PR and using that as a reference to undo changes to any mention of geventconnpool. Geventconnpool, when set as a requirement without the reference portion, is defaulting to get a commit from 2014 which then breaks the code, as we want the newest version from them. -4. Run `docker-compose build` to build a new image for local development with the updated dependencies. +5. Run `docker-compose build` to build a new image for local development with the updated dependencies. The reason for de-coupling the `build` and `lock` steps is to increase consistency between builds--a run of `build` will always get exactly the dependencies listed in `Pipfile.lock`, nothing more, nothing less. \ No newline at end of file diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index a27199cdb..1a33c2791 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -71,17 +71,23 @@ class Command(BaseCommand): # Secret is encrypted into getgov-credentials # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when ready - # encrypted_metadata is the encrypted output + # Encrypt the metadata + # TODO: UPDATE SECRET_ENCRYPT_METADATA pw getgov-credentials on stable encrypted_metadata = self._encrypt_metadata(s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA)) print("encrypted_metadata is:", encrypted_metadata) - + print("the type is: ", type(encrypted_metadata)) # Send the metadata file that is zipped - # Q: Would we set the vars I set in email.py here to pass in to the helper function or best way to invoke - # send_templated_email(encrypted_metadata, attachment=True) - + # TODO: Make new .txt files + send_templated_email( + "emails/metadata_body.txt", + "emails/metadata_subject.txt", + to_address="rebecca.hsieh@truss.works", # TODO: Update to settings.DEFAULT_FROM_EMAIL once tested + file=encrypted_metadata, + ) + def _encrypt_metadata(self, input_file, output_file, password): - # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities - # Could also use compression=pyzipper.ZIP_LZMA? + # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster + # We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size with pyzipper.AESZipFile(output_file, 'w', compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES) as f_out: f_out.setpassword(password) f_out.writestr('encrypted_metadata.txt', input_file) diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 199a6c304..5f3e42eb5 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -4,6 +4,10 @@ import boto3 import logging from django.conf import settings from django.template.loader import get_template +from email.mime.base import MIMEBase +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText logger = logging.getLogger(__name__) @@ -15,7 +19,7 @@ class EmailSendingError(RuntimeError): pass -def send_templated_email(template_name: str, subject_template_name: str, to_address: str, context={}): +def send_templated_email(template_name: str, subject_template_name: str, to_address: str, context={}, file: str=None): """Send an email built from a template to one email address. template_name and subject_template_name are relative to the same template @@ -40,74 +44,56 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr except Exception as exc: raise EmailSendingError("Could not access the SES client.") from exc - # Are we okay with passing in "attachment" var in as boolean parameter - # If so, TODO: add attachment boolean to other functions try: - #if not attachment: - ses_client.send_email( - FromEmailAddress=settings.DEFAULT_FROM_EMAIL, - Destination={"ToAddresses": [to_address]}, - Content={ - "Simple": { - "Subject": {"Data": subject}, - "Body": {"Text": {"Data": email_body}}, + if file is None: + ses_client.send_email( + FromEmailAddress=settings.DEFAULT_FROM_EMAIL, + Destination={"ToAddresses": [to_address]}, + Content={ + "Simple": { + "Subject": {"Data": subject}, + "Body": {"Text": {"Data": email_body}}, + }, }, - }, - ) - # else: # has attachment - # same as above but figure out how to attach a file - # via boto3 "boto3 SES file attachment" - # we also want this to only send to the help email - - # from email.mime.multipart import MIMEMultipart - # from email.mime.text import MIMEText - # from email.mime.application import MIMEApplication + ) + if file is not None: + # TODO: Update sender email when we figure out + ses_client = boto3.client( + "ses", + region_name=settings.AWS_REGION, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=settings.BOTO_CONFIG, + ) - # sender_email = 'sender@example.com' - # recipient_email = 'help@get.gov' - # subject = 'DOTGOV-Full Domain Metadata' - # body = 'Domain metadata email, should have an attachment included change here later.' - # attachment_path = 'path/to/attachment/file.pdf' - # aws_region = 'sesv2' - - # response = send_email_with_attachment(sender_email, recipient_email, subject, body, attachment_path, aws_region) - # print(response) + #TODO: Update sender to settings.DEFAULT_FROM_EMAIL + response = send_email_with_attachment(settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, file, ses_client) + print("Response from send_email_with_attachment_is:", response) except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc +def send_email_with_attachment(sender, recipient, subject, body, attachment_file, ses_client): + # Create a multipart/mixed parent container + msg = MIMEMultipart('mixed') + msg['Subject'] = subject + msg['From'] = sender + msg['To'] = recipient -# def send_email_with_attachment(sender, recipient, subject, body, attachment_path, aws_region): - # # Create a multipart/mixed parent container - # msg = MIMEMultipart('mixed') - # msg['Subject'] = subject - # msg['From'] = sender_email - # msg['To'] = recipient_email + # Add the text part + text_part = MIMEText(body, 'plain') + msg.attach(text_part) - # # Add the text part - # text_part = MIMEText(body, 'plain') - # msg.attach(text_part) + # Add the attachment part - # # Add the attachment part - # with open(attachment_path, 'rb') as attachment_file: - # attachment_data = attachment_file.read() - # attachment_part = MIMEApplication(attachment_data) - # attachment_part.add_header('Content-Disposition', f'attachment; filename="{attachment_path}"') - # msg.attach(attachment_part) + # set it into this "type" + attachment_part = MIMEApplication(attachment_file) + # Adding attachment header + filename that the attachment will be called + attachment_part.add_header('Content-Disposition', f'attachment; filename="encrypted_metadata.zip"') + msg.attach(attachment_part) - # # Send the email - # response = ses_client.send_raw_email( - # Source=sender, - # Destinations=[recipient], - # RawMessage={'Data': msg.as_string()} - # ) - - # ses_client.send_email( - # FromEmailAddress=settings.DEFAULT_FROM_EMAIL, - # Destination={"ToAddresses": [to_address]}, - # Content={ - # "Simple": { - # "Subject": {"Data": subject}, - # "Body": {"Text": {"Data": email_body}}, - # }, - # }, - # ) \ No newline at end of file + response = ses_client.send_raw_email( + Source=sender, + Destinations=[recipient], + RawMessage={"Data": msg.as_string()} + ) + return response From 4b1b1386a8911718887d826950a006853ef46dbd Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 1 Mar 2024 13:44:52 -0500 Subject: [PATCH 017/125] clean up JS, init dates --- src/registrar/assets/js/get-gov-admin.js | 39 -------------------- src/registrar/templates/admin/analytics.html | 32 +++++----------- 2 files changed, 10 insertions(+), 61 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 618cc284c..2c4d4d854 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -307,45 +307,6 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, viewLink.setAttribute('title', viewLink.getAttribute('title-template').replace('selected item', elementText)); } -/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, - * attach the seleted start and end dates to a url that'll trigger the view, and finally - * redirect to that url. -*/ -(function (){ - - // Get the current date in the format YYYY-MM-DD - let currentDate = new Date().toISOString().split('T')[0]; - - // Default the value of the start date input field to the current date - let startDateInput =document.getElementById('start'); - - // Default the value of the end date input field to the current date - let endDateInput =document.getElementById('end'); - - let exportButtons = document.querySelectorAll('.exportLink'); - - if (exportButtons.length > 0) { - startDateInput.value = currentDate; - endDateInput.value = currentDate; - - exportButtons.forEach((btn) => { - btn.addEventListener('click', function() { - // Get the selected start and end dates - let startDate = startDateInput.value; - let endDate = endDateInput.value; - let exportUrl = btn.dataset.exportUrl; - - // Build the URL with parameters - exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; - - // Redirect to the export URL - window.location.href = exportUrl; - }); - }); - } - -})(); - /** An IIFE for admin in DjangoAdmin to listen to changes on the domain request * status select amd to show/hide the rejection reason */ diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 735386d38..6eb26307c 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -12,7 +12,7 @@

At a glance

-
+
-

You should probably remove these domains from the registry instead of deleting them.

+

You should probably remove these domains from the registry instead.

This action cannot be undone.

From 7e4dc38b40e7ce92cf9ed426f49a7e4c6ebc345d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:50:03 -0700 Subject: [PATCH 026/125] Change bullet list style --- src/registrar/assets/sass/_theme/_admin.scss | 2 +- .../templates/django/admin/domain_delete_confirmation.html | 2 +- .../django/admin/domain_delete_selected_confirmation.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 9f6db0c46..88675cb32 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -312,7 +312,7 @@ input.admin-confirm-button { max-width: 68ex; } -.django-admin-custom-bullets { +.django-admin-custom-bullets ul > li { // Set list-style-type to inherit without modifying text size list-style-type: inherit; } diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html index 5a9bef5b0..2836c32f7 100644 --- a/src/registrar/templates/django/admin/domain_delete_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -11,7 +11,7 @@

When a domain is deleted:

-
+
  • The domain will no longer appear in the registrar / admin.
  • It will be removed from the registry.
  • diff --git a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html index 3e0a32a4d..6872ea9af 100644 --- a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html @@ -12,7 +12,7 @@

    When a domain is deleted:

    -
    +
    • The domain will no longer appear in the registrar / admin.
    • It will be removed from the registry.
    • From cc1555ab9a76f42455cd995d8cf45daeda1fb785 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:08:52 -0700 Subject: [PATCH 027/125] Unit tests --- src/registrar/assets/sass/_theme/_admin.scss | 5 ----- .../django/admin/domain_delete_confirmation.html | 2 +- .../domain_delete_selected_confirmation.html | 2 +- src/registrar/tests/test_admin.py | 15 ++++++--------- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 88675cb32..cbec4d1f2 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -312,11 +312,6 @@ input.admin-confirm-button { max-width: 68ex; } -.django-admin-custom-bullets ul > li { - // Set list-style-type to inherit without modifying text size - list-style-type: inherit; -} - .usa-summary-box__dhs-color { color: $dhs-blue-70; } diff --git a/src/registrar/templates/django/admin/domain_delete_confirmation.html b/src/registrar/templates/django/admin/domain_delete_confirmation.html index 2836c32f7..5a9bef5b0 100644 --- a/src/registrar/templates/django/admin/domain_delete_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_confirmation.html @@ -11,7 +11,7 @@

      When a domain is deleted:

      -
      +
      • The domain will no longer appear in the registrar / admin.
      • It will be removed from the registry.
      • diff --git a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html index 6872ea9af..3e0a32a4d 100644 --- a/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html +++ b/src/registrar/templates/django/admin/domain_delete_selected_confirmation.html @@ -12,7 +12,7 @@

        When a domain is deleted:

        -
        +
        • The domain will no longer appear in the registrar / admin.
        • It will be removed from the registry.
        • diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 3c5861ee1..f85396f10 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -241,7 +241,7 @@ class TestDomainAdmin(MockEppLib, WebTest): # click the "Manage" link confirmation_page = domain_change_page.click("Delete", index=0) - content_slice = "

          When a domain is removed from the registry:

          " + content_slice = "When a domain is deleted:" self.assertContains(confirmation_page, content_slice) def test_short_org_name_in_domains_list(self): @@ -350,7 +350,7 @@ class TestDomainAdmin(MockEppLib, WebTest): extra_tags="", fail_silently=False, ) - + # The modal should still exist self.assertContains(response, "Are you sure you want to remove this domain from the registry?") self.assertContains(response, "When a domain is removed from the registry:") @@ -364,7 +364,7 @@ class TestDomainAdmin(MockEppLib, WebTest): """ with less_console_noise(): domain = create_ready_domain() - + response = self.app.get(reverse("admin:registrar_domain_change", args=[domain.pk])) # Check the contents of the modal @@ -389,7 +389,7 @@ class TestDomainAdmin(MockEppLib, WebTest): self.assertEqual(response.status_code, 200) self.assertContains(response, domain.name) self.assertContains(response, "Remove hold") - + # The modal should still exist # Check for the header self.assertContains(response, "Are you sure you want to place this domain on hold?") @@ -1180,8 +1180,7 @@ class TestDomainApplicationAdmin(MockEppLib): # Create a mock request request = self.factory.post( - "/admin/registrar/domainapplication/{}/change/".format(application.pk), - follow=True + "/admin/registrar/domainapplication/{}/change/".format(application.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): @@ -1214,7 +1213,6 @@ class TestDomainApplicationAdmin(MockEppLib): self.assertEqual(response.status_code, 200) self.assertContains(response, application.requested_domain.name) - # Check that the modal has the right content # Check for the header self.assertContains(response, "Are you sure you want to select ineligible status?") @@ -1227,8 +1225,7 @@ class TestDomainApplicationAdmin(MockEppLib): # Create a mock request request = self.factory.post( - "/admin/registrar/domainapplication/{}/change/".format(application.pk), - follow=True + "/admin/registrar/domainapplication/{}/change/".format(application.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): # Modify the application's property From 747af8834791b1fa1fee8c0694088eb3964fbef2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:18:59 -0700 Subject: [PATCH 028/125] Add missing unit test + linting --- src/registrar/tests/common.py | 28 ++++++++++++++++++++++++---- src/registrar/tests/test_admin.py | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index ee1ab8b68..1825d38fd 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -97,7 +97,7 @@ def less_console_noise(output_stream=None): class GenericTestHelper(TestCase): """A helper class that contains various helper functions for TestCases""" - def __init__(self, admin, model=None, url=None, user=None, factory=None, **kwargs): + def __init__(self, admin, model=None, url=None, user=None, factory=None, client=None, **kwargs): """ Parameters: admin (ModelAdmin): The Django ModelAdmin instance associated with the model. @@ -112,6 +112,7 @@ class GenericTestHelper(TestCase): self.admin = admin self.model = model self.url = url + self.client = client def assert_table_sorted(self, o_index, sort_fields): """ @@ -147,9 +148,7 @@ class GenericTestHelper(TestCase): dummy_request.user = self.user # Mock a user request - middleware = SessionMiddleware(lambda req: req) - middleware.process_request(dummy_request) - dummy_request.session.save() + dummy_request = self._mock_user_request_for_factory(dummy_request) expected_sort_order = list(self.model.objects.order_by(*sort_fields)) @@ -160,6 +159,27 @@ class GenericTestHelper(TestCase): self.assertEqual(expected_sort_order, returned_sort_order) + def _mock_user_request_for_factory(self, request): + """Adds sessionmiddleware when using factory to associate session information""" + middleware = SessionMiddleware(lambda req: req) + middleware.process_request(request) + request.session.save() + return request + + def get_table_delete_confirmation_page(self, selected_across: str, index: str): + """ + Grabs the response for the delete confirmation page (generated from the actions toolbar). + selected_across and index must both be numbers encoded as str, e.g. "0" rather than 0 + """ + + response = self.client.post( + self.url, + {"action": "delete_selected", "select_across": selected_across, "index": index, "_selected_action": "23"}, + follow=True, + ) + print(f"what is the response? {response}") + return response + class MockUserLogin: def __init__(self, get_response): diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index f85396f10..7e032ff5c 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -61,6 +61,16 @@ class TestDomainAdmin(MockEppLib, WebTest): self.factory = RequestFactory() self.app.set_user(self.superuser.username) self.client.force_login(self.superuser) + + # Contains some test tools + self.test_helper = GenericTestHelper( + factory=self.factory, + user=self.superuser, + admin=self.admin, + url=reverse("admin:registrar_domain_changelist"), + model=Domain, + client=self.client, + ) super().setUp() @skip("TODO for another ticket. This test case is grabbing old db data.") @@ -244,6 +254,21 @@ class TestDomainAdmin(MockEppLib, WebTest): content_slice = "When a domain is deleted:" self.assertContains(confirmation_page, content_slice) + def test_custom_delete_confirmation_page_table(self): + """Tests if we override the delete confirmation page for custom content on the table""" + # Create a ready domain + domain, _ = Domain.objects.get_or_create(name="fake.gov", state=Domain.State.READY) + + # Get the index. The post expects the index to be encoded as a string + index = f"{domain.id}" + + # Simulate selecting a single record, then clicking "Delete selected domains" + response = self.test_helper.get_table_delete_confirmation_page("0", index) + + # Check that our content exists + content_slice = "When a domain is deleted:" + self.assertContains(response, content_slice) + def test_short_org_name_in_domains_list(self): """ Make sure the short name is displaying in admin on the list page From c89d76fb47fb9da7f40d0a29669a2d13020d7cf2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 4 Mar 2024 17:24:53 -0500 Subject: [PATCH 029/125] WIP unit testing --- src/registrar/admin.py | 49 ++- src/registrar/assets/sass/_theme/_admin.scss | 3 + src/registrar/models/domain.py | 2 +- src/registrar/templates/admin/analytics.html | 17 +- src/registrar/templates/admin/app_list.html | 2 +- src/registrar/tests/data/mocks.py | 232 ++++++++++++ src/registrar/tests/test_admin_views.py | 4 +- src/registrar/tests/test_reports.py | 370 ++++++++++++------- src/registrar/utility/csv_export.py | 109 +++--- 9 files changed, 547 insertions(+), 241 deletions(-) create mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 262cebd18..5bf41777b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -374,8 +374,8 @@ def analytics(request): avg_approval_time = last_30_days_approved_applications.annotate( approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] - # format the timedelta? - avg_approval_time = str(avg_approval_time) + # Format the timedelta to display only days + avg_approval_time = f"{avg_approval_time.days} days" start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -383,75 +383,69 @@ def analytics(request): start_date_formatted = csv_export.format_start_date(start_date) end_date_formatted = csv_export.format_end_date(end_date) - # Managed vs Unmanaged filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) - # Ready and Deleted domains filter_ready_domains_start_date = { "domain__state__in": [Domain.State.READY], "domain__first_ready__lte": start_date_formatted, } + filter_ready_domains_end_date = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": start_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - - filter_ready_domains_end_date = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - } - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) - filter_deleted_domains_end_date = { "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": start_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - - filter_requests_end_date = { - "created_at__lte": end_date_formatted, - } - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) - filter_submitted_requests_end_date = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) context = dict( @@ -459,6 +453,7 @@ def analytics(request): data=dict( user_count=models.User.objects.all().count(), domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.all().filter(state=models.Domain.State.READY).count(), last_30_days_applications=last_30_days_applications.count(), last_30_days_approved_applications=last_30_days_approved_applications.count(), average_application_approval_time_last_30_days=avg_approval_time, @@ -1096,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1448,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] + readonly_fields = ["state", "expiration_date", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index dad88b6a4..29d0e3b2a 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -319,6 +319,9 @@ input.admin-confirm-button { .usa-icon { top: 2px; } + a.button:active, a.button:focus { + text-decoration: none; + } } .module--custom { diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..3b18ac8b6 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper): first_ready = DateField( null=True, - editable=False, + editable=True, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 29faffd3b..380922845 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -16,6 +16,7 @@
          • User Count: {{ data.user_count }}
          • Domain Count: {{ data.domain_count }}
          • +
          • Domains in READY state: {{ data.ready_domain_count }}
          • Domain applications (last 30 days): {{ data.last_30_days_applications }}
          • Approved applications (last 30 days): {{ data.last_30_days_approved_applications }}
          • Average approval time for applications (last 30 days): {{ data.average_application_approval_time_last_30_days }}
          • @@ -63,8 +64,6 @@ {% comment %} Inputs of type date suck for accessibility. We'll need to replace those guys with a django form once we figure out how to hook one onto this page. - The challenge is in the path definition in urls. It does NOT like admin/export_domain_growth/ - See the commit "Review for ticket #999" {% endcomment %}
            @@ -107,7 +106,7 @@
          • -
          -
          - -
          +
          + +
          diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index c96f29a31..4ee2befef 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -71,5 +71,5 @@

          Analytics

          - Dashboard + Dashboard
          \ No newline at end of file diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py new file mode 100644 index 000000000..e6dccb14f --- /dev/null +++ b/src/registrar/tests/data/mocks.py @@ -0,0 +1,232 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation +from registrar.models.domain import Domain +from registrar.models.user_domain_role import UserDomainRole +from registrar.models.public_contact import PublicContact +from registrar.models.user import User +from datetime import date, datetime, timedelta +from django.utils import timezone +from registrar.tests.common import MockEppLib + +class MockDb(MockEppLib): + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + # Test for more than 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + # Test for just 1 domain manager + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + # self.domain_request_1, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # requested_domain=self.domain_1.name, + # organization_type="federal", + # federal_agency="World War I Centennial Commission", + # federal_type="executive", + # is_election_board=True + # ) + # self.domain_request_2, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_2, + # organization_type="interstate", + # is_election_board=True + # ) + # self.domain_request_3, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_3, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_4, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_4, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=True + # ) + # self.domain_request_5, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_5, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_6, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_6, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_7, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_7, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_request_8, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_8, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_9, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_9, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + # self.domain_information_10, _ = DomainApplication.objects.get_or_create( + # creator=self.user, + # domain=self.domain_10, + # organization_type="federal", + # federal_agency="Armed Forces Retirement Home", + # is_election_board=False + # ) + + def tearDown(self): + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() + super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py index e55175db9..cc4b3f1c7 100644 --- a/src/registrar/tests/test_admin_views.py +++ b/src/registrar/tests/test_admin_views.py @@ -3,7 +3,7 @@ from django.urls import reverse from registrar.tests.common import create_superuser -class TestViews(TestCase): +class TestAdminViews(TestCase): def setUp(self): self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -26,7 +26,7 @@ class TestViews(TestCase): # Construct the URL for the export data view with start_date and end_date parameters: # This stuff is currently done in JS - export_data_url = reverse("admin:admin_export_domain_growth") + f"?start_date={start_date}&end_date={end_date}" + export_data_url = reverse("export_domains_growth") + f"?start_date={start_date}&end_date={end_date}" # Make a GET request to the export data page response = self.client.get(export_data_url) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index c00c2b221..43efb3128 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -8,9 +8,12 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.common import MockEppLib +from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - write_csv, + format_end_date, + format_start_date, + get_sliced_domains, + write_domains_csv, get_default_start_date, get_default_end_date, ) @@ -231,136 +234,11 @@ class CsvReportsTest(TestCase): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockEppLib): +class ExportDataTest(MockDb): def setUp(self): super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() super().tearDown() def test_export_domains_to_writer_security_emails(self): @@ -403,7 +281,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) @@ -427,7 +305,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_csv(self): + def test_write_domains_csv(self): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" @@ -462,7 +340,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -486,7 +364,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_additional(self): + def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" with less_console_noise(): # Create a CSV file in memory @@ -512,7 +390,7 @@ class ExportDataTest(MockEppLib): ], } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True ) # Reset the CSV file's position to the beginning @@ -535,7 +413,7 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) - def test_write_body_with_date_filter_pulls_domains_in_range(self): + def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): """Test that domains that are 1. READY and their first_ready dates are in range 2. DELETED and their deleted dates are in range @@ -546,7 +424,7 @@ class ExportDataTest(MockEppLib): and would have been easy to set up, but expected_content would contain created_at dates which are hard to mock. - TODO: Simplify is created_at is not needed for the report.""" + TODO: Simplify if created_at is not needed for the report.""" with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -591,7 +469,7 @@ class ExportDataTest(MockEppLib): } # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -599,7 +477,7 @@ class ExportDataTest(MockEppLib): get_domain_managers=False, should_write_header=True, ) - write_csv( + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -664,7 +542,7 @@ class ExportDataTest(MockEppLib): } self.maxDiff = None # Call the export functions - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True ) @@ -677,11 +555,11 @@ class ExportDataTest(MockEppLib): expected_content = ( "Domain name,Status,Expiration date,Domain type,Agency," "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager email 1,Domain manager email 2,\n" + "Security contact email,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "adomain10.gov,Ready,,Federal,Armed Forces Retirement Home,,,, , ,\n" "adomain2.gov,Dns needed,,Interstate,,,,, , , ,meoward@rocks.com\n" "cdomain1.gov,Ready,,Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,info@example.com\n" + ", , , ,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" "ddomain3.gov,On hold,,Federal,Armed Forces Retirement Home,,,, , , ,,\n" ) # Normalize line endings and remove commas, @@ -690,6 +568,210 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + def test_export_data_managed_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + # Call the export functions + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_managed_domains_end_date, + get_domain_managers=True, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,1\n" + "\n" + "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" + "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_export_data_unmanaged_domains_to_csv(self): + """""" + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # Call the export functions + writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + write_domains_csv( + writer, + columns, + sort_fields, + filter_unmanaged_domains_end_date, + get_domain_managers=False, + should_write_header=True, + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + self.maxDiff=None + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) + + def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): + """Test that requests that are + 1. SUBMITTED and their submission_date are in range + are pulled when the growth report conditions are applied to export_requests_to_writed. + Test that requests are sorted by requested domain name. + """ + + pass class HelperFunctions(TestCase): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -704,3 +786,11 @@ class HelperFunctions(TestCase): expected_date = timezone.now() actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) + + def get_sliced_domains(self): + """Should get fitered domains counts sliced by org type and election office.""" + pass + + def test_get_sliced_requests(self): + """Should get fitered requests counts sliced by org type and election office.""" + pass \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index bec5f3835..cbdbfddb3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -25,9 +25,10 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") + DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) + .distinct() ) # Do a mass concat of the first and last name fields for authorizing_official. @@ -44,7 +45,7 @@ def get_domain_infos(filter_condition, sort_fields): return domain_infos_cleaned -def parse_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): +def parse_domain_row(columns, domain_info: DomainInformation, security_emails_dict=None, get_domain_managers=False): """Given a set of columns, generate a new row from cleaned column data""" # Domain should never be none when parsing this information @@ -136,7 +137,7 @@ def update_columns_with_domain_managers(columns, max_dm_count): columns.append(f"Domain manager email {i}") -def write_csv( +def write_domains_csv( writer, columns, sort_fields, @@ -145,8 +146,8 @@ def write_csv( should_write_header=True, ): """ - Receives params from the parent methods and outputs a CSV with fltered and sorted domains. - Works with write_header as longas the same writer object is passed. + Receives params from the parent methods and outputs a CSV with filtered and sorted domains. + Works with write_header as long as the same writer object is passed. get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice """ @@ -172,7 +173,7 @@ def write_csv( rows = [] for domain_info in page.object_list: try: - row = parse_row(columns, domain_info, security_emails_dict, get_domain_managers) + row = parse_domain_row(columns, domain_info, security_emails_dict, get_domain_managers) rows.append(row) except ValueError: # This should not happen. If it does, just skip this row. @@ -188,7 +189,6 @@ def write_csv( def get_requests(filter_condition, sort_fields): requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) - return requests @@ -235,7 +235,8 @@ def write_requests_csv( filter_condition, should_write_header=True, ): - """ """ + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" all_requetsts = get_requests(filter_condition, sort_fields) @@ -295,7 +296,7 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) def export_data_full_to_csv(csv_file): @@ -326,7 +327,7 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def export_data_federal_to_csv(csv_file): @@ -358,7 +359,7 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) def get_default_start_date(): @@ -426,8 +427,8 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) - write_csv( + write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( writer, columns, sort_fields_for_deleted_domains, @@ -440,19 +441,19 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): def get_sliced_domains(filter_condition): """Get fitered domains counts sliced by org type and election office.""" - domains = DomainInformation.objects.all().filter(**filter_condition) + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() domains_count = domains.count() - federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() + federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() state_or_territory = domains.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = domains.filter(is_election_board=True).count() + ).distinct().count() + tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = domains.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -471,19 +472,19 @@ def get_sliced_domains(filter_condition): def get_sliced_requests(filter_condition): """Get fitered requests counts sliced by org type and election office.""" - requests = DomainApplication.objects.all().filter(**filter_condition) + requests = DomainApplication.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).count() - interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() + federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() state_or_territory = requests.filter( organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).count() - tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).count() - county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).count() - city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).count() - election_board = requests.filter(is_election_board=True).count() + ).distinct().count() + tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() + special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + election_board = requests.filter(is_election_board=True).distinct().count() return [ requests_count, @@ -500,7 +501,8 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get domains have domain managers for two different dates.""" + """Get counts for domains that have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -512,14 +514,13 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): sort_fields = [ "domain__name", ] - filter_managed_domains_start_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT SRAT DATE"]) + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( [ "Total", @@ -537,16 +538,6 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_managed_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, @@ -571,7 +562,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, @@ -582,7 +573,8 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get domains that do not have domain managers for two different dates.""" + """Get counts for domains that do not have domain managers for two different dates, + get list of domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -619,16 +611,6 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - write_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_start_date, - get_domain_managers=True, - should_write_header=True, - ) - writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, @@ -653,18 +635,23 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_csv( + write_domains_csv( writer, columns, sort_fields, filter_unmanaged_domains_end_date, - get_domain_managers=True, + get_domain_managers=False, should_write_header=True, ) def export_data_requests_growth_to_csv(csv_file, start_date, end_date): - """ """ + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_requests_body SUBMITTED requests that are created between + the start and end dates. Specify sort params. + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -676,7 +663,7 @@ def export_data_requests_growth_to_csv(csv_file, start_date, end_date): "Submission date", ] sort_fields = [ - # "domain__name", + "requested_domain__name", ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, From f427076598e42c27b9f4bb65fb9b6181dbb88f60 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Mon, 4 Mar 2024 16:45:05 -0800 Subject: [PATCH 030/125] Fix emailing functionality and update subject and body and file names --- .../generate_current_metadata_report.py | 59 +++++++++++-------- .../templates/emails/metadata_body.txt | 1 + .../templates/emails/metadata_subject.txt | 2 + src/registrar/utility/email.py | 37 ++++++------ 4 files changed, 55 insertions(+), 44 deletions(-) create mode 100644 src/registrar/templates/emails/metadata_body.txt create mode 100644 src/registrar/templates/emails/metadata_subject.txt diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 1a33c2791..023a19f10 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -4,6 +4,8 @@ import logging import os import pyzipper +from datetime import datetime + from django.core.management import BaseCommand from django.conf import settings from registrar.utility import csv_export @@ -13,9 +15,11 @@ from ...utility.email import send_templated_email, EmailSendingError logger = logging.getLogger(__name__) + class Command(BaseCommand): help = ( - "Generates and uploads a current-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." + "Generates and uploads a current-metadata.csv file to our S3 bucket " + "which is based off of all existing Domains." ) def add_arguments(self, parser): @@ -26,7 +30,7 @@ class Command(BaseCommand): default=True, help="Flag that determines if we do a check for os.path.exists. Used for test cases", ) - + def handle(self, **options): """Grabs the directory then creates current-metadata.csv in that directory""" file_name = "current-metadata.csv" @@ -58,38 +62,43 @@ class Command(BaseCommand): # Upload this generated file for our S3 instance s3_client.upload_file(file_path, file_name) - """ - We want to make sure to upload to s3 for back up - And now we also want to get the file and encrypt it so we can send it in an email - """ - # Encrypt metadata into a zip file + # Set zip file name + current_date = datetime.now().strftime("%m%d%Y") + current_filename = f"domain-metadata-{current_date}.zip" + # Pre-set zip file name + encrypted_metadata_output = current_filename - # pre-setting zip file name - encrypted_metadata_output = 'encrypted_metadata.zip' + # Set context for the subject + current_date_str = datetime.now().strftime("%Y-%m-%d") - # Secret is encrypted into getgov-credentials # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when ready - - # Encrypt the metadata - # TODO: UPDATE SECRET_ENCRYPT_METADATA pw getgov-credentials on stable - encrypted_metadata = self._encrypt_metadata(s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA)) - print("encrypted_metadata is:", encrypted_metadata) - print("the type is: ", type(encrypted_metadata)) + + # Encrypt the metadata + encrypted_metadata_in_bytes = self._encrypt_metadata( + s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA) + ) + # Send the metadata file that is zipped - # TODO: Make new .txt files send_templated_email( - "emails/metadata_body.txt", - "emails/metadata_subject.txt", - to_address="rebecca.hsieh@truss.works", # TODO: Update to settings.DEFAULT_FROM_EMAIL once tested - file=encrypted_metadata, + template_name="emails/metadata_body.txt", + subject_template_name="emails/metadata_subject.txt", + to_address=settings.DEFAULT_FROM_EMAIL, + # to_address="rebecca.hsieh@truss.works ", # TODO: Update to settings.DEFAULT_FROM_EMAIL once tested + context={"current_date_str": current_date_str}, + file=encrypted_metadata_in_bytes, ) def _encrypt_metadata(self, input_file, output_file, password): + current_date = datetime.now().strftime("%m%d%Y") + current_filename = f"domain-metadata-{current_date}.txt" # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster # We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size - with pyzipper.AESZipFile(output_file, 'w', compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES) as f_out: + with pyzipper.AESZipFile( + output_file, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES + ) as f_out: f_out.setpassword(password) - f_out.writestr('encrypted_metadata.txt', input_file) - return output_file - + f_out.writestr(current_filename, input_file) + with open(output_file, "rb") as file_data: + attachment_in_bytes = file_data.read() + return attachment_in_bytes diff --git a/src/registrar/templates/emails/metadata_body.txt b/src/registrar/templates/emails/metadata_body.txt new file mode 100644 index 000000000..adf0a186c --- /dev/null +++ b/src/registrar/templates/emails/metadata_body.txt @@ -0,0 +1 @@ +An export of all .gov metadata. diff --git a/src/registrar/templates/emails/metadata_subject.txt b/src/registrar/templates/emails/metadata_subject.txt new file mode 100644 index 000000000..5fdece7ef --- /dev/null +++ b/src/registrar/templates/emails/metadata_subject.txt @@ -0,0 +1,2 @@ +Domain metadata - {{current_date_str}} + diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 5f3e42eb5..a81a41716 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -2,6 +2,7 @@ import boto3 import logging +from datetime import datetime from django.conf import settings from django.template.loader import get_template from email.mime.base import MIMEBase @@ -19,7 +20,7 @@ class EmailSendingError(RuntimeError): pass -def send_templated_email(template_name: str, subject_template_name: str, to_address: str, context={}, file: str=None): +def send_templated_email(template_name: str, subject_template_name: str, to_address: str, context={}, file: str = None): """Send an email built from a template to one email address. template_name and subject_template_name are relative to the same template @@ -56,8 +57,7 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr }, }, ) - if file is not None: - # TODO: Update sender email when we figure out + else: ses_client = boto3.client( "ses", region_name=settings.AWS_REGION, @@ -65,35 +65,34 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, config=settings.BOTO_CONFIG, ) - - #TODO: Update sender to settings.DEFAULT_FROM_EMAIL - response = send_email_with_attachment(settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, file, ses_client) + # Define the subject line with the current date + response = send_email_with_attachment( + settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, file, ses_client + ) + # TODO: Remove this print statement print("Response from send_email_with_attachment_is:", response) except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc + def send_email_with_attachment(sender, recipient, subject, body, attachment_file, ses_client): # Create a multipart/mixed parent container - msg = MIMEMultipart('mixed') - msg['Subject'] = subject - msg['From'] = sender - msg['To'] = recipient + msg = MIMEMultipart("mixed") + msg["Subject"] = subject + msg["From"] = sender + msg["To"] = recipient # Add the text part - text_part = MIMEText(body, 'plain') + text_part = MIMEText(body, "plain") msg.attach(text_part) # Add the attachment part - - # set it into this "type" attachment_part = MIMEApplication(attachment_file) # Adding attachment header + filename that the attachment will be called - attachment_part.add_header('Content-Disposition', f'attachment; filename="encrypted_metadata.zip"') + current_date = datetime.now().strftime("%m%d%Y") + current_filename = f"domain-metadata-{current_date}.zip" + attachment_part.add_header("Content-Disposition", f'attachment; filename="{current_filename}"') msg.attach(attachment_part) - response = ses_client.send_raw_email( - Source=sender, - Destinations=[recipient], - RawMessage={"Data": msg.as_string()} - ) + response = ses_client.send_raw_email(Source=sender, Destinations=[recipient], RawMessage={"Data": msg.as_string()}) return response From efab77d39f98b3a10c5328af3963c59fd74c5e3b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:58:20 -0700 Subject: [PATCH 031/125] Basic Ao check --- src/registrar/models/domain.py | 1 + src/registrar/views/domain.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 449c4c4bb..5f17a5332 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -198,6 +198,7 @@ class Domain(TimeStampedModel, DomainHelper): is called in the validate function on the request/domain page throws- RegistryError or InvalidDomainError""" + if not cls.string_could_be_domain(domain): logger.warning("Not a valid domain: %s" % str(domain)) # throw invalid domain error so that it can be caught in diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 72eb65f1e..eec90aeb5 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -10,6 +10,7 @@ import logging from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError +from django.forms import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse @@ -22,6 +23,8 @@ from registrar.models import ( User, UserDomainRole, ) +from registrar.models.domain_application import DomainApplication +from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact from registrar.utility.enums import DefaultEmail from registrar.utility.errors import ( @@ -225,6 +228,35 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" + # if not self.request.user.is_staff: + + _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) + + current_domain_info = None + if _domain_info.exists() and _domain_info.count() == 1: + current_domain_info = _domain_info.get() + else: + logger.error("Could not update Authorizing Official. No domain info exists, or duplicates exist.") + messages.error(self.request, "Something went wrong when attempting to save.") + return self.form_invalid(form) + + # Determine if the domain is federal or tribal + is_federal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # Get the old and new ao values + old_authorizing_official = form.initial + new_authorizing_official = form.cleaned_data + + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, but it won't always be. + if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: + logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") + messages.error(self.request, "You cannot modify the Authorizing Official.") + + return self.form_invalid(form) + # Set the domain information in the form so that it can be accessible # to associate a new Contact as authorizing official, if new Contact is needed # in the save() method From ff6149a1c42ff89ee9a5b7e8ef6a3010bb97d4df Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 12:51:22 -0500 Subject: [PATCH 032/125] Unit tests plus revert hacks to add data --- src/registrar/admin.py | 4 +- src/registrar/models/domain.py | 2 +- src/registrar/signals.py | 2 - src/registrar/tests/data/mocks.py | 84 ++++------------------ src/registrar/tests/test_reports.py | 106 ++++++++++++++++++++++++++-- src/registrar/utility/csv_export.py | 8 +-- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5bf41777b..78f85f0f9 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1091,7 +1091,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "submission_date", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), ( "Type of organization", { @@ -1443,7 +1443,7 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" - readonly_fields = ["state", "expiration_date", "deleted"] + readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 3b18ac8b6..449c4c4bb 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1022,7 +1022,7 @@ class Domain(TimeStampedModel, DomainHelper): first_ready = DateField( null=True, - editable=True, + editable=False, help_text="The last time this domain moved into the READY state", ) diff --git a/src/registrar/signals.py b/src/registrar/signals.py index 74dc8a063..4e7768ef4 100644 --- a/src/registrar/signals.py +++ b/src/registrar/signals.py @@ -27,7 +27,6 @@ def handle_profile(sender, instance, **kwargs): last_name = getattr(instance, "last_name", "") email = getattr(instance, "email", "") phone = getattr(instance, "phone", "") - logger.info(f"in handle_profile first {instance}") is_new_user = kwargs.get("created", False) @@ -37,7 +36,6 @@ def handle_profile(sender, instance, **kwargs): contacts = Contact.objects.filter(user=instance) if len(contacts) == 0: # no matching contact - logger.info(f"inside no matching contacts for first {first_name} last {last_name} email {email}") Contact.objects.create( user=instance, first_name=first_name, diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py index e6dccb14f..25f56f247 100644 --- a/src/registrar/tests/data/mocks.py +++ b/src/registrar/tests/data/mocks.py @@ -1,5 +1,6 @@ from django.test import TestCase from django.contrib.auth import get_user_model +from api.tests.common import less_console_noise from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain @@ -8,7 +9,7 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from datetime import date, datetime, timedelta from django.utils import timezone -from registrar.tests.common import MockEppLib +from registrar.tests.common import MockEppLib, completed_application class MockDb(MockEppLib): def setUp(self): @@ -152,81 +153,22 @@ class MockDb(MockEppLib): user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER ) - # self.domain_request_1, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # requested_domain=self.domain_1.name, - # organization_type="federal", - # federal_agency="World War I Centennial Commission", - # federal_type="executive", - # is_election_board=True - # ) - # self.domain_request_2, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_2, - # organization_type="interstate", - # is_election_board=True - # ) - # self.domain_request_3, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_3, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_4, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_4, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=True - # ) - # self.domain_request_5, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_5, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_6, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_6, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_7, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_7, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_request_8, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_8, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_9, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_9, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) - # self.domain_information_10, _ = DomainApplication.objects.get_or_create( - # creator=self.user, - # domain=self.domain_10, - # organization_type="federal", - # federal_agency="Armed Forces Retirement Home", - # is_election_board=False - # ) + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() def tearDown(self): PublicContact.objects.all().delete() Domain.objects.all().delete() DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() User.objects.all().delete() UserDomainRole.objects.all().delete() super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 43efb3128..cc7cc7991 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -2,6 +2,7 @@ import csv import io from django.test import Client, RequestFactory, TestCase from io import StringIO +from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain from registrar.models.public_contact import PublicContact @@ -13,9 +14,11 @@ from registrar.utility.csv_export import ( format_end_date, format_start_date, get_sliced_domains, + get_sliced_requests, write_domains_csv, get_default_start_date, get_default_end_date, + write_requests_csv, ) from django.core.management import call_command @@ -27,7 +30,7 @@ import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import less_console_noise +from .common import completed_application, less_console_noise class CsvReportsTest(TestCase): @@ -771,9 +774,58 @@ class ExportDataTest(MockDb): Test that requests are sorted by requested domain name. """ - pass + with less_console_noise(): + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + # We use timezone.make_aware to sync to server time a datetime object with the current date + # (using date.today()) and a specific time (using datetime.min.time()). + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) -class HelperFunctions(TestCase): + # Define columns, sort fields, and filter condition + columns = [ + "Requested domain", + "Organization type", + "Submission date", + ] + sort_fields = [ + "requested_domain__name", + ] + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + "submission_date__gte": start_date, + } + write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name + # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name + expected_content = ( + "Requested domain,Organization type,Submission date\n" + "city3.gov,Federal - Executive,2024-03-05\n" + "city4.gov,Federal - Executive,2024-03-05\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + +class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" def test_get_default_start_date(self): @@ -787,10 +839,52 @@ class HelperFunctions(TestCase): actual_date = get_default_end_date() self.assertEqual(actual_date.date(), expected_date.date()) - def get_sliced_domains(self): + def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - pass + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + + expected_content = ( + [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] + ) + + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - pass \ No newline at end of file + with less_console_noise(): + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + end_date = current_date + timedelta(days=2) + start_date = current_date - timedelta(days=2) + + filter_condition = { + "status": DomainApplication.ApplicationStatus.SUBMITTED, + "submission_date__lte": end_date, + } + submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + + print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') + + expected_content = ( + [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + ) + + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index cbdbfddb3..e09258022 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -188,7 +188,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields) + requests = DomainApplication.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -197,12 +197,8 @@ def parse_request_row(columns, request: DomainApplication): requested_domain_name = "No requested domain" - # Domain should never be none when parsing this information if request.requested_domain is not None: - domain = request.requested_domain - requested_domain_name = domain.name - - domain = request.requested_domain # type: ignore + requested_domain_name = request.requested_domain.name if request.federal_type: request_type = f"{request.get_organization_type_display()} - {request.get_federal_type_display()}" From 795a4d71a298d93fdba43407efff9a65def21fb2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:56:31 -0700 Subject: [PATCH 033/125] Check for agency and org name --- src/registrar/views/domain.py | 68 ++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index eec90aeb5..3fa6a96a2 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -10,7 +10,6 @@ import logging from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError -from django.forms import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse @@ -19,12 +18,12 @@ from django.conf import settings from registrar.models import ( Domain, + DomainApplication, + DomainInformation, DomainInvitation, User, UserDomainRole, ) -from registrar.models.domain_application import DomainApplication -from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact from registrar.utility.enums import DefaultEmail from registrar.utility.errors import ( @@ -136,6 +135,20 @@ class DomainFormBaseView(DomainBaseView, FormMixin): # superclass has the redirect return super().form_invalid(form) + + def get_domain_info_from_domain(self) -> DomainInformation | None: + """ + Grabs the underlying domain_info object based off of self.object.name. + Returns None if nothing is found. + """ + _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) + current_domain_info = None + if _domain_info.exists() and _domain_info.count() == 1: + current_domain_info = _domain_info.get() + else: + logger.error("Could get domain_info. No domain info exists, or duplicates exist.") + + return current_domain_info class DomainView(DomainBaseView): @@ -200,6 +213,43 @@ class DomainOrgNameAddressView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the organization name and mailing address.""" + + current_domain_info = self.get_domain_info_from_domain() + if current_domain_info is None: + messages.error(self.request, "Something went wrong when attempting to save.") + return self.form_invalid(form) + + # Get the old and new values to see if a change is occuring + old_org_info = form.initial + new_org_info = form.cleaned_data + + if old_org_info != new_org_info: + + error_message = None + # These actions, aside from the default, should be blocked by the UI, as the field is readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, but it won't always be. + match current_domain_info.organization_type: + case DomainApplication.OrganizationChoices.FEDERAL: + old_fed_agency = old_org_info.get("federal_agency", None) + new_fed_agency = new_org_info.get("federal_agency", None) + if old_fed_agency != new_fed_agency: + error_message = "You cannot modify Federal Agency" + case DomainApplication.OrganizationChoices.TRIBAL: + old_org_name = old_org_info.get("organization_name", None) + new_org_name = new_org_info.get("organization_name", None) + if old_org_name != new_org_name: + error_message = "You cannot modify Organization Name." + case _: + # Do nothing + pass + + # If we encounter an error, forbid this action. + if error_message is not None: + logger.warning(f"User {self.request.user} attempted to change org info on {self.object.name}") + messages.error(self.request, "You cannot modify the Authorizing Official.") + return self.form_invalid(form) + form.save() messages.success(self.request, "The organization information for this domain has been updated.") @@ -229,14 +279,9 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" # if not self.request.user.is_staff: - - _domain_info = DomainInformation.objects.filter(domain__name=self.object.name) - - current_domain_info = None - if _domain_info.exists() and _domain_info.count() == 1: - current_domain_info = _domain_info.get() - else: - logger.error("Could not update Authorizing Official. No domain info exists, or duplicates exist.") + + current_domain_info = self.get_domain_info_from_domain() + if current_domain_info is None: messages.error(self.request, "Something went wrong when attempting to save.") return self.form_invalid(form) @@ -254,7 +299,6 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) # Set the domain information in the form so that it can be accessible From 091e4c900e9853fe2600af2e0fb72e3d1a30ae80 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:28:12 -0500 Subject: [PATCH 034/125] Clean up common mock class and how it's inherited --- src/registrar/tests/common.py | 245 ++++++++++++++++++++++++---- src/registrar/tests/data/mocks.py | 174 -------------------- src/registrar/tests/test_reports.py | 166 +++++-------------- src/registrar/utility/csv_export.py | 4 +- 4 files changed, 251 insertions(+), 338 deletions(-) delete mode 100644 src/registrar/tests/data/mocks.py diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index ee1ab8b68..9666d135d 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -13,6 +13,8 @@ from django.contrib.sessions.middleware import SessionMiddleware from django.conf import settings from django.contrib.auth import get_user_model, login from django.utils.timezone import make_aware +from datetime import date, datetime, timedelta +from django.utils import timezone from registrar.models import ( Contact, @@ -35,6 +37,7 @@ from epplibwrapper import ( ErrorCode, responses, ) +from registrar.models.user_domain_role import UserDomainRole from registrar.models.utility.contact_error import ContactError, ContactErrorCodes @@ -470,6 +473,176 @@ class AuditedAdminMockData: application.alternative_domains.add(alt) return application + +class MockDb(TestCase): + """Hardcoded mocks make test case assertions sraightforward.""" + + def setUp(self): + super().setUp() + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + # Create a time-aware current date + current_datetime = timezone.now() + # Extract the date part + current_date = current_datetime.date() + # Create start and end dates using timedelta + self.end_date = current_date + timedelta(days=2) + self.start_date = current_date - timedelta(days=2) + + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + federal_type="executive", + is_election_board=True + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + is_election_board=True + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=True + ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + is_election_board=False + ) + + meoward_user = get_user_model().objects.create( + username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" + ) + + lebowski_user = get_user_model().objects.create( + username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER + ) + + _, created = UserDomainRole.objects.get_or_create( + user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER + ) + + with less_console_noise(): + self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") + self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") + self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") + self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") + self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_3.submit() + self.domain_request_3.save() + self.domain_request_4.submit() + self.domain_request_4.save() + + def tearDown(self): + super().tearDown() + PublicContact.objects.all().delete() + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainApplication.objects.all().delete() + User.objects.all().delete() + UserDomainRole.objects.all().delete() def mock_user(): @@ -645,7 +818,7 @@ class MockEppLib(TestCase): self, id, email, - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), pw="thisisnotapassword", ): fake = info.InfoContactResultData( @@ -683,82 +856,82 @@ class MockEppLib(TestCase): mockDataInfoDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meoward.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainAndIPAddress = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoDomainNotSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.meow.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataInfoDomainSubdomainNoIP = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.subdomainwoip.gov"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockDataExtensionDomain = fakedEppObject( "fakePw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[common.DomainContact(contact="123", type=PublicContact.ContactTypeChoices.SECURITY)], hosts=["fake.host.com"], statuses=[ common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], - ex_date=datetime.date(2023, 11, 15), + ex_date=date(2023, 11, 15), ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "123", "123@mail.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithContacts = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -783,7 +956,7 @@ class MockEppLib(TestCase): InfoDomainWithDefaultSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultSec", @@ -798,11 +971,11 @@ class MockEppLib(TestCase): ) mockVerisignDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( - "defaultVeri", "registrar@dotgov.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" + "defaultVeri", "registrar@dotgov.gov", datetime(2023, 5, 25, 19, 45, 35), "lastPw" ) InfoDomainWithVerisignSecurityContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultVeri", @@ -818,7 +991,7 @@ class MockEppLib(TestCase): InfoDomainWithDefaultTechnicalContact = fakedEppObject( "fakepw", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="defaultTech", @@ -843,14 +1016,14 @@ class MockEppLib(TestCase): infoDomainNoContact = fakedEppObject( "security", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["fake.host.com"], ) infoDomainThreeHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.my-nameserver-1.com", @@ -861,43 +1034,43 @@ class MockEppLib(TestCase): infoDomainNoHost = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[], ) infoDomainTwoHosts = fakedEppObject( "my-nameserver.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=["ns1.my-nameserver-1.com", "ns1.my-nameserver-2.com"], ) mockDataInfoHosts = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="1.2.3.4"), common.Ip(addr="2.3.4.5")], ) mockDataInfoHosts1IP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35)), addrs=[common.Ip(addr="2.0.0.8")], ) mockDataInfoHostsNotSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 26, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 26, 19, 45, 35)), addrs=[], ) mockDataInfoHostsSubdomainNoIP = fakedEppObject( "lastPw", - cr_date=make_aware(datetime.datetime(2023, 8, 27, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 8, 27, 19, 45, 35)), addrs=[], ) - mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime.datetime(2023, 8, 25, 19, 45, 35))) + mockDataHostChange = fakedEppObject("lastPw", cr_date=make_aware(datetime(2023, 8, 25, 19, 45, 35))) addDsData1 = { "keyTag": 1234, "alg": 3, @@ -929,7 +1102,7 @@ class MockEppLib(TestCase): infoDomainHasIP = fakedEppObject( "nameserverwithip.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -954,7 +1127,7 @@ class MockEppLib(TestCase): justNameserver = fakedEppObject( "justnameserver.com", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[ common.DomainContact( contact="securityContact", @@ -977,7 +1150,7 @@ class MockEppLib(TestCase): infoDomainCheckHostIPCombo = fakedEppObject( "nameserversubdomain.gov", - cr_date=make_aware(datetime.datetime(2023, 5, 25, 19, 45, 35)), + cr_date=make_aware(datetime(2023, 5, 25, 19, 45, 35)), contacts=[], hosts=[ "ns1.nameserversubdomain.gov", @@ -987,27 +1160,27 @@ class MockEppLib(TestCase): mockRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2023, 5, 25), + ex_date=date(2023, 5, 25), ) mockButtonRenewedDomainExpDate = fakedEppObject( "fake.gov", - ex_date=datetime.date(2025, 5, 25), + ex_date=date(2025, 5, 25), ) mockDnsNeededRenewedDomainExpDate = fakedEppObject( "fakeneeded.gov", - ex_date=datetime.date(2023, 2, 15), + ex_date=date(2023, 2, 15), ) mockMaximumRenewedDomainExpDate = fakedEppObject( "fakemaximum.gov", - ex_date=datetime.date(2024, 12, 31), + ex_date=date(2024, 12, 31), ) mockRecentRenewedDomainExpDate = fakedEppObject( "waterbutpurple.gov", - ex_date=datetime.date(2024, 11, 15), + ex_date=date(2024, 11, 15), ) def _mockDomainName(self, _name, _avail=False): diff --git a/src/registrar/tests/data/mocks.py b/src/registrar/tests/data/mocks.py deleted file mode 100644 index 25f56f247..000000000 --- a/src/registrar/tests/data/mocks.py +++ /dev/null @@ -1,174 +0,0 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model -from api.tests.common import less_console_noise -from registrar.models.domain_application import DomainApplication -from registrar.models.domain_information import DomainInformation -from registrar.models.domain import Domain -from registrar.models.user_domain_role import UserDomainRole -from registrar.models.public_contact import PublicContact -from registrar.models.user import User -from datetime import date, datetime, timedelta -from django.utils import timezone -from registrar.tests.common import MockEppLib, completed_application - -class MockDb(MockEppLib): - def setUp(self): - super().setUp() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create( - name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() - ) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_5, _ = Domain.objects.get_or_create( - name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) - ) - self.domain_6, _ = Domain.objects.get_or_create( - name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) - ) - self.domain_7, _ = Domain.objects.get_or_create( - name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - self.domain_8, _ = Domain.objects.get_or_create( - name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() - ) - # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) - # and a specific time (using datetime.min.time()). - # Deleted yesterday - self.domain_9, _ = Domain.objects.get_or_create( - name="zdomain9.gov", - state=Domain.State.DELETED, - deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), - ) - # ready tomorrow - self.domain_10, _ = Domain.objects.get_or_create( - name="adomain10.gov", - state=Domain.State.READY, - first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), - ) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - is_election_board=True - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=True - ) - self.domain_information_5, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_5, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_6, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_6, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_7, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_7, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_8, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_8, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_9, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_9, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - self.domain_information_10, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_10, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - is_election_board=False - ) - - meoward_user = get_user_model().objects.create( - username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com" - ) - - lebowski_user = get_user_model().objects.create( - username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co" - ) - - # Test for more than 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=self.user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - _, created = UserDomainRole.objects.get_or_create( - user=lebowski_user, domain=self.domain_1, role=UserDomainRole.Roles.MANAGER - ) - - # Test for just 1 domain manager - _, created = UserDomainRole.objects.get_or_create( - user=meoward_user, domain=self.domain_2, role=UserDomainRole.Roles.MANAGER - ) - - with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") - self.domain_request_3.submit() - self.domain_request_3.save() - self.domain_request_4.submit() - self.domain_request_4.save() - - def tearDown(self): - PublicContact.objects.all().delete() - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - DomainApplication.objects.all().delete() - User.objects.all().delete() - UserDomainRole.objects.all().delete() - super().tearDown() \ No newline at end of file diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cc7cc7991..e6230fadb 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,14 +5,9 @@ from io import StringIO from registrar.models.domain_application import DomainApplication from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain -from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model -from registrar.models.user_domain_role import UserDomainRole -from registrar.tests.data.mocks import MockDb from registrar.utility.csv_export import ( - format_end_date, - format_start_date, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -30,60 +25,17 @@ import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore from datetime import date, datetime, timedelta from django.utils import timezone -from .common import completed_application, less_console_noise +from .common import MockDb, MockEppLib, less_console_noise -class CsvReportsTest(TestCase): +class CsvReportsTest(MockDb): """Tests to determine if we are uploading our reports correctly""" def setUp(self): """Create fake domain data""" + super().setUp() self.client = Client(HTTP_HOST="localhost:8080") self.factory = RequestFactory() - username = "test_user" - first_name = "First" - last_name = "Last" - email = "info@example.com" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email - ) - - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - - self.domain_information_1, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_1, - organization_type="federal", - federal_agency="World War I Centennial Commission", - federal_type="executive", - ) - self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - ) - self.domain_information_3, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_3, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - self.domain_information_4, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_4, - organization_type="federal", - federal_agency="Armed Forces Retirement Home", - ) - - def tearDown(self): - """Delete all faked data""" - Domain.objects.all().delete() - DomainInformation.objects.all().delete() - User.objects.all().delete() - super().tearDown() @boto3_mocking.patching def test_generate_federal_report(self): @@ -94,6 +46,7 @@ class CsvReportsTest(TestCase): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -114,6 +67,7 @@ class CsvReportsTest(TestCase): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), + call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -172,6 +126,7 @@ class CsvReportsTest(TestCase): @boto3_mocking.patching def test_load_federal_report(self): """Tests the get_current_federal api endpoint""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -205,6 +160,7 @@ class CsvReportsTest(TestCase): @boto3_mocking.patching def test_load_full_report(self): """Tests the current-federal api link""" + with less_console_noise(): mock_client = MagicMock() mock_client_instance = mock_client.return_value @@ -237,7 +193,7 @@ class CsvReportsTest(TestCase): self.assertEqual(expected_file_content, response.content) -class ExportDataTest(MockDb): +class ExportDataTest(MockDb, MockEppLib): def setUp(self): super().setUp() @@ -247,6 +203,7 @@ class ExportDataTest(MockDb): def test_export_domains_to_writer_security_emails(self): """Test that export_domains_to_writer returns the expected security email""" + with less_console_noise(): # Add security email information self.domain_1.name = "defaultsecurity.gov" @@ -312,6 +269,7 @@ class ExportDataTest(MockDb): """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -369,6 +327,7 @@ class ExportDataTest(MockDb): def test_write_domains_body_additional(self): """An additional test for filters and multi-column sort""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -428,15 +387,11 @@ class ExportDataTest(MockDb): which are hard to mock. TODO: Simplify if created_at is not needed for the report.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) - # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -460,15 +415,15 @@ class ExportDataTest(MockDb): "domain__state__in": [ Domain.State.READY, ], - "domain__first_ready__lte": end_date, - "domain__first_ready__gte": start_date, + "domain__first_ready__lte": self.end_date, + "domain__first_ready__gte": self.start_date, } filter_conditions_for_deleted_domains = { "domain__state__in": [ Domain.State.DELETED, ], - "domain__deleted__lte": end_date, - "domain__deleted__gte": start_date, + "domain__deleted__lte": self.end_date, + "domain__deleted__gte": self.start_date, } # Call the export functions @@ -515,13 +470,13 @@ class ExportDataTest(MockDb): def test_export_domains_to_writer_domain_managers(self): """Test that export_domains_to_writer returns the - expected domain managers""" + expected domain managers.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition - columns = [ "Domain name", "Status", @@ -572,13 +527,13 @@ class ExportDataTest(MockDb): self.assertEqual(csv_content, expected_content) def test_export_data_managed_domains_to_csv(self): - """""" + """Test get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -589,7 +544,7 @@ class ExportDataTest(MockDb): ] filter_managed_domains_start_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) # Call the export functions @@ -610,13 +565,11 @@ class ExportDataTest(MockDb): ) writer.writerow(managed_domains_sliced_at_start_date) writer.writerow([]) - filter_managed_domains_end_date = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -634,7 +587,6 @@ class ExportDataTest(MockDb): ) writer.writerow(managed_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -647,9 +599,7 @@ class ExportDataTest(MockDb): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" @@ -663,20 +613,22 @@ class ExportDataTest(MockDb): "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): - """""" + """Test get counts for domains that do not have domain managers for two different dates, + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) - start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -687,7 +639,7 @@ class ExportDataTest(MockDb): ] filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date, + "domain__first_ready__lte": self.start_date, } unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) # Call the export functions @@ -708,13 +660,11 @@ class ExportDataTest(MockDb): ) writer.writerow(unmanaged_domains_sliced_at_start_date) writer.writerow([]) - filter_unmanaged_domains_end_date = { "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( [ @@ -732,7 +682,6 @@ class ExportDataTest(MockDb): ) writer.writerow(unmanaged_domains_sliced_at_end_date) writer.writerow([]) - write_domains_csv( writer, columns, @@ -745,9 +694,7 @@ class ExportDataTest(MockDb): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" @@ -761,10 +708,12 @@ class ExportDataTest(MockDb): "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -778,17 +727,6 @@ class ExportDataTest(MockDb): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - # We use timezone.make_aware to sync to server time a datetime object with the current date - # (using date.today()) and a specific time (using datetime.min.time()). - - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - # Define columns, sort fields, and filter condition columns = [ "Requested domain", @@ -800,16 +738,14 @@ class ExportDataTest(MockDb): ] filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, - "submission_date__gte": start_date, + "submission_date__lte": self.end_date, + "submission_date__gte": self.start_date, } write_requests_csv(writer, columns, sort_fields, filter_condition, should_write_header=True) # Reset the CSV file's position to the beginning csv_file.seek(0) - # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( @@ -817,12 +753,12 @@ class ExportDataTest(MockDb): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -841,50 +777,28 @@ class HelperFunctions(MockDb): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date, + "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] ) - self.assertEqual(managed_domains_sliced_at_end_date, expected_content) - - def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" + with less_console_noise(): - # Create a time-aware current date - current_datetime = timezone.now() - # Extract the date part - current_date = current_datetime.date() - # Create start and end dates using timedelta - end_date = current_date + timedelta(days=2) - start_date = current_date - timedelta(days=2) - filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, - "submission_date__lte": end_date, + "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - - print(f'managed_domains_sliced_at_end_date {submitted_requests_sliced_at_end_date}') - expected_content = ( [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1bdd3fd82..22467bf6b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -498,7 +498,7 @@ def get_sliced_requests(filter_condition): def export_data_managed_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that have domain managers for two different dates, - get list of domains at end_date.""" + get list of managed domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -570,7 +570,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): """Get counts for domains that do not have domain managers for two different dates, - get list of domains at end_date.""" + get list of unmanaged domains at end_date.""" start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) From 2081f5c56483cf9dadd3f5f71d06204c3055d184 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:41:34 -0500 Subject: [PATCH 035/125] lint --- src/registrar/admin.py | 2 +- src/registrar/tests/common.py | 47 ++++++++++++++----------- src/registrar/tests/test_reports.py | 53 +++++++++++++++-------------- src/registrar/utility/csv_export.py | 46 +++++++++++++++++-------- 4 files changed, 86 insertions(+), 62 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 78f85f0f9..3d6d87367 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -424,7 +424,7 @@ def analytics(request): "domain__state__in": [Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) # Created and Submitted requests diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 9666d135d..e6e642918 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -1,4 +1,3 @@ -import datetime import os import logging @@ -473,7 +472,8 @@ class AuditedAdminMockData: application.alternative_domains.add(alt) return application - + + class MockDb(TestCase): """Hardcoded mocks make test case assertions sraightforward.""" @@ -535,69 +535,66 @@ class MockDb(TestCase): organization_type="federal", federal_agency="World War I Centennial Commission", federal_type="executive", - is_election_board=True + is_election_board=True, ) self.domain_information_2, _ = DomainInformation.objects.get_or_create( - creator=self.user, - domain=self.domain_2, - organization_type="interstate", - is_election_board=True + creator=self.user, domain=self.domain_2, organization_type="interstate", is_election_board=True ) self.domain_information_3, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_3, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_4, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_4, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=True + is_election_board=True, ) self.domain_information_5, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_5, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_6, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_6, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_7, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_7, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_8, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_8, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_9, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_9, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) self.domain_information_10, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_10, organization_type="federal", federal_agency="Armed Forces Retirement Home", - is_election_board=False + is_election_board=False, ) meoward_user = get_user_model().objects.create( @@ -625,11 +622,21 @@ class MockDb(TestCase): ) with less_console_noise(): - self.domain_request_1 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov") - self.domain_request_2 = completed_application(status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov") - self.domain_request_3 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov") - self.domain_request_4 = completed_application(status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov") - self.domain_request_5 = completed_application(status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov") + self.domain_request_1 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city1.gov" + ) + self.domain_request_2 = completed_application( + status=DomainApplication.ApplicationStatus.IN_REVIEW, name="city2.gov" + ) + self.domain_request_3 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city3.gov" + ) + self.domain_request_4 = completed_application( + status=DomainApplication.ApplicationStatus.STARTED, name="city4.gov" + ) + self.domain_request_5 = completed_application( + status=DomainApplication.ApplicationStatus.APPROVED, name="city5.gov" + ) self.domain_request_3.submit() self.domain_request_3.save() self.domain_request_4.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 55f5c9108..03f792825 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -43,7 +43,7 @@ class CsvReportsTest(MockDb): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] # We don't actually want to write anything for a test case, @@ -64,7 +64,7 @@ class CsvReportsTest(MockDb): expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call('adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n'), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), ] @@ -525,7 +525,7 @@ class ExportDataTest(MockDb, MockEppLib): def test_export_data_managed_domains_to_csv(self): """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" + get list of managed domains at end_date.""" with less_console_noise(): # Create a CSV file in memory @@ -596,32 +596,34 @@ class ExportDataTest(MockDb, MockEppLib): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" "1,1,0,0,0,0,0,0,0,1\n" "\n" "Domain name,Domain type,Domain manager email 1,Domain manager email 2,Domain manager email 3\n" "cdomain1.gov,Federal - Executive,meoward@rocks.com,info@example.com,big_lebowski@dude.co\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_export_data_unmanaged_domains_to_csv(self): """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - + get list of unmanaged domains at end_date.""" + with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() @@ -691,26 +693,28 @@ class ExportDataTest(MockDb, MockEppLib): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - self.maxDiff=None + self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( "UNMANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" "UNMANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district,School district,Election office\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" "\n" "Domain name,Domain type\n" "adomain10.gov,Federal\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): @@ -750,14 +754,15 @@ class ExportDataTest(MockDb, MockEppLib): "city3.gov,Federal - Executive,2024-03-05\n" "city4.gov,Federal - Executive,2024-03-05\n" ) - + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - + self.assertEqual(csv_content, expected_content) + class HelperFunctions(MockDb): """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" @@ -774,28 +779,24 @@ class HelperFunctions(MockDb): def test_get_sliced_domains(self): """Should get fitered domains counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) - expected_content = ( - [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] - ) + expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" - + with less_console_noise(): filter_condition = { "status": DomainApplication.ApplicationStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) - expected_content = ( - [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] - ) - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file + expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 0] + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 44e34164d..fdebfef77 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -128,6 +128,7 @@ def _get_security_emails(sec_contact_ids): return security_emails_dict + def write_domains_csv( writer, columns, @@ -253,7 +254,6 @@ def write_requests_csv( logger.error("csv_export -> Error when parsing row, domain was None") continue - if should_write_header: write_header(writer, columns) writer.writerows(rows) @@ -293,7 +293,9 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=True, should_write_header=True + ) def export_data_full_to_csv(csv_file): @@ -324,7 +326,9 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def export_data_federal_to_csv(csv_file): @@ -356,7 +360,9 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) def get_default_start_date(): @@ -424,7 +430,9 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): "domain__deleted__gte": start_date_formatted, } - write_domains_csv(writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True) + write_domains_csv( + writer, columns, sort_fields, filter_condition, get_domain_managers=False, should_write_header=True + ) write_domains_csv( writer, columns, @@ -442,14 +450,18 @@ def get_sliced_domains(filter_condition): domains_count = domains.count() federal = domains.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = domains.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).count() - state_or_territory = domains.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = domains.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = domains.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = domains.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + domains.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = domains.filter(is_election_board=True).distinct().count() return [ @@ -473,14 +485,18 @@ def get_sliced_requests(filter_condition): requests_count = requests.count() federal = requests.filter(organization_type=DomainApplication.OrganizationChoices.FEDERAL).distinct().count() interstate = requests.filter(organization_type=DomainApplication.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = requests.filter( - organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY - ).distinct().count() + state_or_territory = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) tribal = requests.filter(organization_type=DomainApplication.OrganizationChoices.TRIBAL).distinct().count() county = requests.filter(organization_type=DomainApplication.OrganizationChoices.COUNTY).distinct().count() city = requests.filter(organization_type=DomainApplication.OrganizationChoices.CITY).distinct().count() - special_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - school_district = requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + special_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + requests.filter(organization_type=DomainApplication.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) election_board = requests.filter(is_election_board=True).distinct().count() return [ From 05533179b67d380e9c69d39c68bc23703ef2e1de Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:51:35 -0500 Subject: [PATCH 036/125] cleanup --- src/registrar/admin.py | 3 ++- src/registrar/templates/admin/analytics.html | 1 - src/registrar/templates/admin/app_list.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3d6d87367..41391f724 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -364,6 +364,7 @@ class UserContactInline(admin.StackedInline): def analytics(request): + """View for the reports page.""" thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) @@ -377,6 +378,7 @@ def analytics(request): # Format the timedelta to display only days avg_approval_time = f"{avg_approval_time.days} days" + # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") @@ -427,7 +429,6 @@ def analytics(request): deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) - # Created and Submitted requests filter_requests_start_date = { "created_at__lte": start_date_formatted, } diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 380922845..72aa244cf 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -1,7 +1,6 @@ {% extends "admin/base_site.html" %} {% load static %} - {% block content_title %}

          Registrar Analytics

          {% endblock %} {% block content %} diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index 4ee2befef..dd7e27f33 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -72,4 +72,4 @@

          Analytics

          Dashboard -
          \ No newline at end of file +
          From 9ca8322510c185006dc1d43f3468420204fd456a Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 5 Mar 2024 11:52:48 -0800 Subject: [PATCH 037/125] Update docs, csv set up, and add unit test --- .../runbooks/rotate_application_secrets.md | 6 ++++ .../generate_current_metadata_report.py | 8 ++--- src/registrar/tests/test_emails.py | 33 ++++++++++++++++++- src/registrar/utility/email.py | 4 +-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/operations/runbooks/rotate_application_secrets.md b/docs/operations/runbooks/rotate_application_secrets.md index a776e60b8..f7a5004ef 100644 --- a/docs/operations/runbooks/rotate_application_secrets.md +++ b/docs/operations/runbooks/rotate_application_secrets.md @@ -117,3 +117,9 @@ You'll need to give the new certificate to the registry vendor _before_ rotating ## REGISTRY_HOSTNAME This is the hostname at which the registry can be found. + +## SECRET_METADATA_KEY + +This is in reference to the key for the metadata email that is sent daily. Reach out to product team members or leads with access to security passwords if the passcode is needed. + +To change the password, use a password generator to generate a password, then update the user credentials per the above instructions. Be sure to update the `KDBX` file in Google Drive with this password change. diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 023a19f10..2478f9e6b 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -72,7 +72,7 @@ class Command(BaseCommand): # Set context for the subject current_date_str = datetime.now().strftime("%Y-%m-%d") - # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when ready + # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when merging # Encrypt the metadata encrypted_metadata_in_bytes = self._encrypt_metadata( @@ -83,15 +83,15 @@ class Command(BaseCommand): send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - to_address=settings.DEFAULT_FROM_EMAIL, - # to_address="rebecca.hsieh@truss.works ", # TODO: Update to settings.DEFAULT_FROM_EMAIL once tested + # to_address=settings.DEFAULT_FROM_EMAIL, # TODO: Uncomment this when ready to merge + to_address="rebecca.hsieh@truss.works ", context={"current_date_str": current_date_str}, file=encrypted_metadata_in_bytes, ) def _encrypt_metadata(self, input_file, output_file, password): current_date = datetime.now().strftime("%m%d%Y") - current_filename = f"domain-metadata-{current_date}.txt" + current_filename = f"domain-metadata-{current_date}.csv" # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster # We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size with pyzipper.AESZipFile( diff --git a/src/registrar/tests/test_emails.py b/src/registrar/tests/test_emails.py index f2a94a186..292fe5b1c 100644 --- a/src/registrar/tests/test_emails.py +++ b/src/registrar/tests/test_emails.py @@ -5,7 +5,8 @@ from unittest.mock import MagicMock from django.test import TestCase from .common import completed_application, less_console_noise - +from datetime import datetime +from registrar.utility import email import boto3_mocking # type: ignore @@ -182,3 +183,33 @@ class TestEmails(TestCase): self.assertNotIn("Anything else", body) # spacing should be right between adjacent elements self.assertRegex(body, r"5557\n\n----") + + @boto3_mocking.patching + def test_send_email_with_attachment(self): + with boto3_mocking.clients.handler_for("ses", self.mock_client_class): + sender_email = "sender@example.com" + recipient_email = "recipient@example.com" + subject = "Test Subject" + body = "Test Body" + attachment_file = b"Attachment file content" + current_date = datetime.now().strftime("%m%d%Y") + current_filename = f"domain-metadata-{current_date}.zip" + + response = email.send_email_with_attachment( + sender_email, recipient_email, subject, body, attachment_file, self.mock_client + ) + print("response is", response) + # Assert that the `send_raw_email` method of the mocked SES client was called with the expected params + self.mock_client.send_raw_email.assert_called_once() + + # Get the args passed to the `send_raw_email` method + call_args = self.mock_client.send_raw_email.call_args[1] + print("call_args is", call_args) + + # Assert that the attachment filename is correct + self.assertEqual(call_args["RawMessage"]["Data"].count(f'filename="{current_filename}"'), 1) + + # Assert that the attachment content is encrypted + self.assertIn("Content-Type: application/octet-stream", call_args["RawMessage"]["Data"]) + self.assertIn("Content-Transfer-Encoding: base64", call_args["RawMessage"]["Data"]) + self.assertIn("Content-Disposition: attachment;", call_args["RawMessage"]["Data"]) diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index a81a41716..ddd211041 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -65,11 +65,11 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, config=settings.BOTO_CONFIG, ) - # Define the subject line with the current date response = send_email_with_attachment( settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, file, ses_client ) - # TODO: Remove this print statement + # TODO: Remove this print statement when ready to merge, + # leaving rn for getting error codes in case print("Response from send_email_with_attachment_is:", response) except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc From 11f69454c14869c28a95e6a3fd7a85578200a262 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:53:32 -0500 Subject: [PATCH 038/125] cleanup --- src/registrar/views/admin_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index c3769ad03..04f98a2c4 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -77,7 +77,7 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) @@ -92,7 +92,7 @@ class ExportDataUnmanagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = ( - f'attachment; filename="managed-vs-unamanaged-domains-{start_date}-to-{end_date}.csv"' + f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' ) csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) From cf97531102df74ec00655f61e6d87a5216086b1b Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 5 Mar 2024 11:56:20 -0800 Subject: [PATCH 039/125] Fix missing brackets --- src/registrar/utility/email.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 8960be001..72bbbabc7 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -59,7 +59,8 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr "Simple": { "Subject": {"Data": subject}, "Body": {"Text": {"Data": email_body}}, - }, + }, + } ) else: ses_client = boto3.client( From 62cf4ecb687077a2c857f53fd2b426c9e3c08b32 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 14:59:37 -0500 Subject: [PATCH 040/125] lint --- src/registrar/views/admin_views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04f98a2c4..04fcaa6f2 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -76,9 +76,7 @@ class ExportDataManagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) return response @@ -91,9 +89,7 @@ class ExportDataUnmanagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = ( - f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - ) + response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From a9878a00730ca2cadf20872207d86ea74729b64d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:01:27 -0500 Subject: [PATCH 041/125] Charts js --- src/registrar/assets/js/get-gov-reports.js | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/registrar/assets/js/get-gov-reports.js diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js new file mode 100644 index 000000000..e900fabe8 --- /dev/null +++ b/src/registrar/assets/js/get-gov-reports.js @@ -0,0 +1,117 @@ +/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, + * attach the seleted start and end dates to a url that'll trigger the view, and finally + * redirect to that url. + * + * This function also sets the start and end dates to match the url params if they exist +*/ +(function () { + // Function to get URL parameter value by name + function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); + } + + // Get the current date in the format YYYY-MM-DD + let currentDate = new Date().toISOString().split('T')[0]; + + // Default the value of the start date input field to the current date + let startDateInput = document.getElementById('start'); + + // Default the value of the end date input field to the current date + let endDateInput = document.getElementById('end'); + + let exportButtons = document.querySelectorAll('.exportLink'); + + if (exportButtons.length > 0) { + // Check if start and end dates are present in the URL + let urlStartDate = getParameterByName('start_date'); + let urlEndDate = getParameterByName('end_date'); + + // Set input values based on URL parameters or current date + startDateInput.value = urlStartDate || currentDate; + endDateInput.value = urlEndDate || currentDate; + + exportButtons.forEach((btn) => { + btn.addEventListener('click', function () { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = btn.dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); + }); + } + +})(); + +document.addEventListener("DOMContentLoaded", function () { + createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); + createComparativeColumnChart("myChart6", "All requests", "Start Date", "End Date"); +}); + +function createComparativeColumnChart(canvasId, title, labelOne, labelTwo) { + var canvas = document.getElementById(canvasId); + var ctx = canvas.getContext("2d"); + + var listOne = JSON.parse(canvas.getAttribute('data-list-one')); + var listTwo = JSON.parse(canvas.getAttribute('data-list-two')); + + var data = { + labels: ["Total", "Federal", "Interstate", "State/Territory", "Tribal", "County", "City", "Special District", "School District", "Election Board"], + datasets: [ + { + label: labelOne, + backgroundColor: "rgba(255, 99, 132, 0.2)", + borderColor: "rgba(255, 99, 132, 1)", + borderWidth: 1, + data: listOne, + }, + { + label: labelTwo, + backgroundColor: "rgba(75, 192, 192, 0.2)", + borderColor: "rgba(75, 192, 192, 1)", + borderWidth: 1, + data: listTwo, + }, + ], + }; + + var options = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'top', + }, + title: { + display: true, + text: title + } + }, + scales: { + y: { + beginAtZero: true, + }, + }, + }; + + new Chart(ctx, { + type: "bar", + data: data, + options: options, + }); +} \ No newline at end of file From d61e17a4adcb70ce38b7b8145325027540cbc38c Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 5 Mar 2024 12:03:11 -0800 Subject: [PATCH 042/125] Fix conflicts and linting --- .../commands/generate_current_metadata_report.py | 2 +- src/registrar/tests/test_emails.py | 2 -- src/registrar/utility/email.py | 7 ++++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 2478f9e6b..a3c7f70e1 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -10,7 +10,7 @@ from django.core.management import BaseCommand from django.conf import settings from registrar.utility import csv_export from registrar.utility.s3_bucket import S3ClientHelper -from ...utility.email import send_templated_email, EmailSendingError +from ...utility.email import send_templated_email logger = logging.getLogger(__name__) diff --git a/src/registrar/tests/test_emails.py b/src/registrar/tests/test_emails.py index 292fe5b1c..2c480cec2 100644 --- a/src/registrar/tests/test_emails.py +++ b/src/registrar/tests/test_emails.py @@ -198,13 +198,11 @@ class TestEmails(TestCase): response = email.send_email_with_attachment( sender_email, recipient_email, subject, body, attachment_file, self.mock_client ) - print("response is", response) # Assert that the `send_raw_email` method of the mocked SES client was called with the expected params self.mock_client.send_raw_email.assert_called_once() # Get the args passed to the `send_raw_email` method call_args = self.mock_client.send_raw_email.call_args[1] - print("call_args is", call_args) # Assert that the attachment filename is correct self.assertEqual(call_args["RawMessage"]["Data"].count(f'filename="{current_filename}"'), 1) diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 72bbbabc7..2fb08d10c 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -5,7 +5,6 @@ import logging from datetime import datetime from django.conf import settings from django.template.loader import get_template -from email.mime.base import MIMEBase from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -20,7 +19,9 @@ class EmailSendingError(RuntimeError): pass -def send_templated_email(template_name: str, subject_template_name: str, to_address: str, bcc_address="", context={}, file: str = None): +def send_templated_email( + template_name: str, subject_template_name: str, to_address: str, bcc_address="", context={}, file: str = None +): """Send an email built from a template to one email address. template_name and subject_template_name are relative to the same template @@ -60,7 +61,7 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr "Subject": {"Data": subject}, "Body": {"Text": {"Data": email_body}}, }, - } + }, ) else: ses_client = boto3.client( From 20161fe7f64c1c0ea215e8c01aa1f3551190f5eb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:18:59 -0700 Subject: [PATCH 043/125] Simplify logic --- src/registrar/forms/domain.py | 75 ++++++++++++++++++++++++++++++----- src/registrar/views/domain.py | 61 +--------------------------- 2 files changed, 65 insertions(+), 71 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 1669774ae..3081cf7c3 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,9 +1,9 @@ """Forms for domain management.""" - +import logging from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator from django.forms import formset_factory - +from registrar.models import DomainApplication from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.utility.errors import ( NameserverError, @@ -23,6 +23,9 @@ from .common import ( import re +logger = logging.getLogger(__name__) + + class DomainAddUserForm(forms.Form): """Form for adding a user to a domain.""" @@ -205,6 +208,13 @@ class ContactForm(forms.ModelForm): "required": "Enter your email address in the required format, like name@example.com." } self.fields["phone"].error_messages["required"] = "Enter your phone number." + self.domainInfo = None + + def set_domain_info(self, domainInfo): + """Set the domain information for the form. + The form instance is associated with the contact itself. In order to access the associated + domain information object, this needs to be set in the form by the view.""" + self.domainInfo = domainInfo class AuthorizingOfficialContactForm(ContactForm): @@ -232,20 +242,32 @@ class AuthorizingOfficialContactForm(ContactForm): self.fields["email"].error_messages = { "required": "Enter an email address in the required format, like name@example.com." } - self.domainInfo = None - - def set_domain_info(self, domainInfo): - """Set the domain information for the form. - The form instance is associated with the contact itself. In order to access the associated - domain information object, this needs to be set in the form by the view.""" - self.domainInfo = domainInfo def save(self, commit=True): - """Override the save() method of the BaseModelForm.""" + """ + Override the save() method of the BaseModelForm. + Used to perform checks on the underlying domain_information object. + If this doesn't exist, we just save as normal. + """ + + # If the underlying Domain doesn't have a domainInfo object, + # just let the default super handle it. + if not self.domainInfo: + return super().save() + + # Determine if the domain is federal or tribal + is_federal = self.domainInfo.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.domainInfo.organization_type == DomainApplication.OrganizationChoices.TRIBAL # Get the Contact object from the db for the Authorizing Official db_ao = Contact.objects.get(id=self.instance.id) - if self.domainInfo and db_ao.has_more_than_one_join("information_authorizing_official"): + + if (is_federal or is_tribal) and self.has_changed(): + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, so lets reserve information for the backend only. + raise ValueError("Authorizing Official cannot be modified for federal or tribal domains.") + elif db_ao.has_more_than_one_join("information_authorizing_official"): # Handle the case where the domain information object is available and the AO Contact # has more than one joined object. # In this case, create a new Contact, and update the new Contact with form data. @@ -254,6 +276,7 @@ class AuthorizingOfficialContactForm(ContactForm): self.domainInfo.authorizing_official = Contact.objects.create(**data) self.domainInfo.save() else: + # If all checks pass, just save normally super().save() @@ -333,6 +356,36 @@ class DomainOrgNameAddressForm(forms.ModelForm): self.fields[field_name].required = True self.fields["state_territory"].widget.attrs.pop("maxlength", None) self.fields["zipcode"].widget.attrs.pop("maxlength", None) + + def save(self, commit=True): + """Override the save() method of the BaseModelForm.""" + if self.has_changed(): + is_federal = self.instance.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.instance.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # This action should be blocked by the UI, as the text fields are readonly. + # If they get past this point, we forbid it this way. + # This could be malicious, so lets reserve information for the backend only. + if is_federal and not self._field_unchanged("federal_agency"): + raise ValueError("federal_agency cannot be modified when the organization_type is federal") + elif is_tribal and not self._field_unchanged("organization_name"): + raise ValueError("organization_name cannot be modified when the organization_type is tribal") + + else: + super().save() + + def _field_unchanged(self, field_name) -> bool: + """ + Checks if a specified field has not changed between the old value + and the new value. + + The old value is grabbed from self.initial. + The new value is grabbed from self.cleaned_data. + """ + old_value = self.initial.get(field_name, None) + new_value = self.cleaned_data.get(field_name, None) + return old_value == new_value + class DomainDnssecForm(forms.Form): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 3fa6a96a2..8598e93fe 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -213,43 +213,6 @@ class DomainOrgNameAddressView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the organization name and mailing address.""" - - current_domain_info = self.get_domain_info_from_domain() - if current_domain_info is None: - messages.error(self.request, "Something went wrong when attempting to save.") - return self.form_invalid(form) - - # Get the old and new values to see if a change is occuring - old_org_info = form.initial - new_org_info = form.cleaned_data - - if old_org_info != new_org_info: - - error_message = None - # These actions, aside from the default, should be blocked by the UI, as the field is readonly. - # If they get past this point, we forbid it this way. - # This could be malicious, but it won't always be. - match current_domain_info.organization_type: - case DomainApplication.OrganizationChoices.FEDERAL: - old_fed_agency = old_org_info.get("federal_agency", None) - new_fed_agency = new_org_info.get("federal_agency", None) - if old_fed_agency != new_fed_agency: - error_message = "You cannot modify Federal Agency" - case DomainApplication.OrganizationChoices.TRIBAL: - old_org_name = old_org_info.get("organization_name", None) - new_org_name = new_org_info.get("organization_name", None) - if old_org_name != new_org_name: - error_message = "You cannot modify Organization Name." - case _: - # Do nothing - pass - - # If we encounter an error, forbid this action. - if error_message is not None: - logger.warning(f"User {self.request.user} attempted to change org info on {self.object.name}") - messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) - form.save() messages.success(self.request, "The organization information for this domain has been updated.") @@ -278,31 +241,9 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): def form_valid(self, form): """The form is valid, save the authorizing official.""" - # if not self.request.user.is_staff: - - current_domain_info = self.get_domain_info_from_domain() - if current_domain_info is None: - messages.error(self.request, "Something went wrong when attempting to save.") - return self.form_invalid(form) - - # Determine if the domain is federal or tribal - is_federal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.FEDERAL - is_tribal = current_domain_info.organization_type == DomainApplication.OrganizationChoices.TRIBAL - - # Get the old and new ao values - old_authorizing_official = form.initial - new_authorizing_official = form.cleaned_data - - # This action should be blocked by the UI, as the text fields are readonly. - # If they get past this point, we forbid it this way. - # This could be malicious, but it won't always be. - if (is_federal or is_tribal) and old_authorizing_official != new_authorizing_official: - logger.warning(f"User {self.request.user} attempted to change AO on {self.object.name}") - messages.error(self.request, "You cannot modify the Authorizing Official.") - return self.form_invalid(form) # Set the domain information in the form so that it can be accessible - # to associate a new Contact as authorizing official, if new Contact is needed + # to associate a new Contact, if a new Contact is needed # in the save() method form.set_domain_info(self.object.domain_info) form.save() From c5f83b937d5e9d4f8c48edf848b0ea148a5e90d3 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 5 Mar 2024 12:21:41 -0800 Subject: [PATCH 044/125] Fix test --- src/registrar/tests/test_emails.py | 2 +- src/registrar/utility/email.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_emails.py b/src/registrar/tests/test_emails.py index 2c480cec2..99db0d644 100644 --- a/src/registrar/tests/test_emails.py +++ b/src/registrar/tests/test_emails.py @@ -195,7 +195,7 @@ class TestEmails(TestCase): current_date = datetime.now().strftime("%m%d%Y") current_filename = f"domain-metadata-{current_date}.zip" - response = email.send_email_with_attachment( + email.send_email_with_attachment( sender_email, recipient_email, subject, body, attachment_file, self.mock_client ) # Assert that the `send_raw_email` method of the mocked SES client was called with the expected params diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 2fb08d10c..35a4ecf03 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -51,11 +51,10 @@ def send_templated_email( destination["BccAddresses"] = [bcc_address] try: - if file is None: ses_client.send_email( FromEmailAddress=settings.DEFAULT_FROM_EMAIL, - Destination={"ToAddresses": [to_address]}, + Destination=destination, Content={ "Simple": { "Subject": {"Data": subject}, From fda2c11fb1a329d1ff03e7108484ccea52947b8d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 5 Mar 2024 15:33:55 -0500 Subject: [PATCH 045/125] Some accessibility work on charts --- src/registrar/admin.py | 2 + src/registrar/assets/js/get-gov-reports.js | 4 +- src/registrar/templates/admin/analytics.html | 54 +++++++++++++++----- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 41391f724..5b8d67983 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -470,6 +470,8 @@ def analytics(request): submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, requests_sliced_at_end_date=requests_sliced_at_end_date, submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, ), ) return render(request, "admin/analytics.html", context) diff --git a/src/registrar/assets/js/get-gov-reports.js b/src/registrar/assets/js/get-gov-reports.js index e900fabe8..d10cf2dc6 100644 --- a/src/registrar/assets/js/get-gov-reports.js +++ b/src/registrar/assets/js/get-gov-reports.js @@ -55,8 +55,8 @@ })(); document.addEventListener("DOMContentLoaded", function () { - createComparativeColumnChart("myChart", "Unmanaged domains", "Start Date", "End Date"); - createComparativeColumnChart("myChart2", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart1", "Managed domains", "Start Date", "End Date"); + createComparativeColumnChart("myChart2", "Unmanaged domains", "Start Date", "End Date"); createComparativeColumnChart("myChart3", "Deleted domains", "Start Date", "End Date"); createComparativeColumnChart("myChart4", "Ready domains", "Start Date", "End Date"); createComparativeColumnChart("myChart5", "Submitted requests", "Start Date", "End Date"); diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 72aa244cf..da7f25c66 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -115,46 +115,76 @@
          - + > +

          Chart: Managed domains

          +

          {{ data.managed_domains_sliced_at_end_date.0 }} managed domains for {{ data.end_date }}

          +
          - + > +

          Chart: Unanaged domains

          +

          {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

          +
          - + > +

          Chart: Deleted domains

          +

          {{ data.deleted_domains_sliced_at_end_date.0 }} deleted domains for {{ data.end_date }}

          +
          - + > +

          Chart: Ready domains

          +

          {{ data.ready_domains_sliced_at_end_date.0 }} ready domains for {{ data.end_date }}

          +
          - + > +

          Chart: Submitted requests

          +

          {{ data.submitted_requests_sliced_at_end_date.0 }} submitted requests for {{ data.end_date }}

          +
          - + > +

          Chart: All requests

          +

          {{ data.requests_sliced_at_end_date.0 }} requests for {{ data.end_date }}

          +
          From 550bed1de6641e4e319b40e98091e7fc150f0d30 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:52:25 -0700 Subject: [PATCH 046/125] Basic styling / HTML content Needs some work, but this is fine for now --- src/registrar/assets/sass/_theme/_forms.scss | 8 ++++++ src/registrar/forms/domain.py | 27 ++++++++++++++++++- .../domain_authorizing_official.html | 9 ++++++- src/registrar/views/domain.py | 12 +++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_forms.scss b/src/registrar/assets/sass/_theme/_forms.scss index 94407f88d..7ebdaca31 100644 --- a/src/registrar/assets/sass/_theme/_forms.scss +++ b/src/registrar/assets/sass/_theme/_forms.scss @@ -38,3 +38,11 @@ legend.float-left-tablet + button.float-right-tablet { margin-top: 1rem; } } + +/* Custom style for disabled inputs */ +// TODO - UPDATE THIS! +.usa-input:disabled, .usa-select:disabled, .usa-textarea:disabled { + background-color: #f0f0f0; + color: #5b616b; + border-color: #5b616b; +} diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 3081cf7c3..f02b0eea1 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -222,7 +222,7 @@ class AuthorizingOfficialContactForm(ContactForm): JOIN = "authorizing_official" - def __init__(self, *args, **kwargs): + def __init__(self, disable_fields=False, *args, **kwargs): super().__init__(*args, **kwargs) # Overriding bc phone not required in this form @@ -243,6 +243,17 @@ class AuthorizingOfficialContactForm(ContactForm): "required": "Enter an email address in the required format, like name@example.com." } + # TODO - uswds text fields dont have disabled styling?? + # All fields should be disabled if the domain is federal or tribal + if disable_fields: + self._mass_disable_fields() + + def _mass_disable_fields(self): + """Given all available fields, invoke .disabled = True on them""" + for field in self.fields.values(): + field.disabled = True + + def save(self, commit=True): """ Override the save() method of the BaseModelForm. @@ -356,6 +367,20 @@ class DomainOrgNameAddressForm(forms.ModelForm): self.fields[field_name].required = True self.fields["state_territory"].widget.attrs.pop("maxlength", None) self.fields["zipcode"].widget.attrs.pop("maxlength", None) + + is_federal = self.instance.organization_type == DomainApplication.OrganizationChoices.FEDERAL + is_tribal = self.instance.organization_type == DomainApplication.OrganizationChoices.TRIBAL + + # (Q) Should required = False be set here? + # These fields should not be None. If they are, + # it seems like an analyst should intervene? + + # TODO - maybe consider adding a modal on these fields on hover + # ALSO TODO - uswds text fields dont have disabled styling?? + if is_federal: + self.fields['federal_agency'].disabled = True + elif is_tribal: + self.fields['organization_name'].disabled = True def save(self, commit=True): """Override the save() method of the BaseModelForm.""" diff --git a/src/registrar/templates/domain_authorizing_official.html b/src/registrar/templates/domain_authorizing_official.html index e7fc12a5e..6768f8b68 100644 --- a/src/registrar/templates/domain_authorizing_official.html +++ b/src/registrar/templates/domain_authorizing_official.html @@ -11,6 +11,13 @@

          Your authorizing official is a person within your organization who can authorize domain requests. This person must be in a role of significant, executive responsibility within the organization. Read more about who can serve as an authorizing official.

          + + {% if organization_type == "federal" or organization_type == "tribal" %} +

          + The authorizing official for your organization can’t be updated here. + To suggest an update, email help@get.gov. +

          + {% endif %} {% include "includes/required_fields.html" %} @@ -24,7 +31,7 @@ {% input_with_errors form.title %} {% input_with_errors form.email %} - + - - + {% if organization_type != "federal" and organization_type != "tribal" %} + + {% endif %} + {% endblock %} {# domain_content #} diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 59b5faaa9..8cc3b16a0 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1087,6 +1087,101 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Not igorville") self.assertContains(success_result_page, "Faketown") + + def test_domain_org_name_address_form_tribal(self): + """ + Submitting a change to organization_name is blocked for tribal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + tribal_org_type = DomainInformation.OrganizationChoices.TRIBAL + self.domain_information.organization_type = tribal_org_type + self.domain_information.save() + try: + # Add an org name + self.domain_information.organization_name = "Town of Igorville" + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") + + self.assertEqual(self.domain_information.organization_type, tribal_org_type) + + org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + print(f"what is the org name page? {org_name_page}") + + form = org_name_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["organization_name"][0] + self.assertEqual(organization_name_input.value, "Town of Igorville") + + # Check if the input field is disabled + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["organization_name"] = "Not igorville" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The org name should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "Town of Igorville") + self.assertNotContains(success_result_page, "Not igorville") + + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") + + def test_domain_org_name_address_form_federal(self): + """ + Submitting a change to federal_agency is blocked for federal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + federal_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = federal_org_type + self.domain_information.save() + try: + # Add a federal agency. Defined as a tuple since this list may change order. + self.domain_information.federal_agency = ("AMTRAK", "AMTRAK") + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") + + self.assertEqual(self.domain_information.organization_type, federal_org_type) + + org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + + form = org_name_page.forms[0] + # Check the value of the input field + federal_agency_input = form.fields["federal_agency"][0] + self.assertEqual(federal_agency_input.value, "AMTRAK") + + # Check if the input field is disabled + self.assertTrue("disabled" in federal_agency_input.attrs) + self.assertEqual(federal_agency_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["organization_name"] = "Not igorville" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The org name should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "Town of Igorville") + self.assertNotContains(success_result_page, "Not igorville") + + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") class TestDomainContactInformation(TestDomainOverview): From ce98272b4841d8186cd53e089827e02bc69f041c Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 6 Mar 2024 11:45:50 -0800 Subject: [PATCH 049/125] Update pipfile --- src/Pipfile | 1 - src/Pipfile.lock | 12 ++++-------- src/requirements.txt | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Pipfile b/src/Pipfile index b40a8c3ea..b9c5d72d4 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -29,7 +29,6 @@ django-login-required-middleware = "*" greenlet = "*" gevent = "*" fred-epplib = {git = "https://github.com/cisagov/epplib.git", ref = "master"} -geventconnpool = {git = "https://github.com/rasky/geventconnpool.git", ref = "1bbb93a714a331a069adf27265fe582d9ba7ecd4"} pyzipper="*" tblib = "*" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 789422c5b..4eb2c0fb3 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8094a1c9461f860e928b51542adf891c0f6f6c4c62bd1bd8ac3bba55a67f918d" + "sha256": "082a951f15bb26a28f2dca7e0840fdf61518b3d90c42d77a310f982344cbd1dc" }, "pipfile-spec": 6, "requires": {}, @@ -458,10 +458,6 @@ "markers": "python_version >= '3.8'", "version": "==24.2.1" }, - "geventconnpool": { - "git": "https://github.com/rasky/geventconnpool.git", - "ref": "1bbb93a714a331a069adf27265fe582d9ba7ecd4" - }, "greenlet": { "hashes": [ "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67", @@ -1217,12 +1213,12 @@ }, "boto3-stubs": { "hashes": [ - "sha256:5ee40bdfba94fcdba26f36869339c849e918827ed1fb2f8e470474e6b1e923ff", - "sha256:cbbae1b811b97e4e1f1d00eba237ff987678e652502226b87e6276f7963935b4" + "sha256:627f8eca69d832581ee1676d39df099a2a2e3a86d6b3ebd21c81c5f11ed6a6fa", + "sha256:a87e7ecbab6235ec371b4363027e57483bca349a9cd5c891f40db81dadfa273e" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.55" + "version": "==1.34.56" }, "botocore": { "hashes": [ diff --git a/src/requirements.txt b/src/requirements.txt index b24e3575a..1db089f5a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -28,7 +28,6 @@ fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a furl==2.1.3 future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' gevent==24.2.1; python_version >= '3.8' -geventconnpool@ git+https://github.com/rasky/geventconnpool.git@1bbb93a714a331a069adf27265fe582d9ba7ecd4 greenlet==3.0.3; python_version >= '3.7' gunicorn==21.2.0; python_version >= '3.5' idna==3.6; python_version >= '3.5' From b50eaf91f623c42bb57a06b49ed43b21c7e07385 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:52:31 -0700 Subject: [PATCH 050/125] Unit tests --- src/registrar/forms/domain.py | 4 +- src/registrar/tests/test_views_domain.py | 213 ++++++++++++++++++++--- src/registrar/views/domain.py | 6 +- 3 files changed, 193 insertions(+), 30 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 3dd2c72ac..5eb043742 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,4 +1,5 @@ """Forms for domain management.""" + import logging from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator @@ -264,8 +265,7 @@ class AuthorizingOfficialContactForm(ContactForm): if disable_maxlength: # Remove the maxlength dialog if "maxlength" in field.widget.attrs: - field.widget.attrs.pop('maxlength', None) - + field.widget.attrs.pop("maxlength", None) def save(self, commit=True): """ diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 8cc3b16a0..2a861a878 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1021,6 +1021,144 @@ class TestDomainAuthorizingOfficial(TestDomainOverview): self.assertEqual("Testy2", self.domain_information.authorizing_official.first_name) self.assertEqual(ao_pk, self.domain_information.authorizing_official.id) + def assert_all_form_fields_have_expected_values(self, form, test_cases, test_for_disabled=False): + """ + Asserts that each specified form field has the expected value and, optionally, checks if the field is disabled. + + This method iterates over a list of tuples, where each + tuple contains a field name and the expected value for that field. + It uses subtests to isolate each assertion, allowing multiple field + checks within a single test method without stopping at the first failure. + + Example usage: + test_cases = [ + ("first_name", "John"), + ("last_name", "Doe"), + ("email", "john.doe@example.com"), + ] + self.assert_all_form_fields_have_expected_values(my_form, test_cases, test_for_disabled=True) + """ + for field_name, expected_value in test_cases: + with self.subTest(field_name=field_name, expected_value=expected_value): + # Test that each field has the value we expect + self.assertEqual(expected_value, form[field_name].value) + + if test_for_disabled: + # Test for disabled on each field + self.assertTrue("disabled" in form[field_name].attrs) + + def test_domain_edit_authorizing_official_federal(self): + """Tests that no edit can occur when the underlying domain is federal""" + + # Set the org type to federal + self.domain_information.organization_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.save() + + # Add an AO. We can do this at the model level, just not the form level. + self.domain_information.authorizing_official = Contact( + first_name="Apple", last_name="Tester", title="CIO", email="nobody@igorville.gov" + ) + self.domain_information.authorizing_official.save() + self.domain_information.save() + + ao_page = self.app.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Test if the form is populating data correctly + ao_form = ao_page.forms[0] + + test_cases = [ + ("first_name", "Apple"), + ("last_name", "Tester"), + ("title", "CIO"), + ("email", "nobody@igorville.gov"), + ] + self.assert_all_form_fields_have_expected_values(ao_form, test_cases, test_for_disabled=True) + + # Attempt to change data on each field. Because this domain is federal, + # this should not succeed. + ao_form["first_name"] = "Orange" + ao_form["last_name"] = "Smoothie" + ao_form["title"] = "Cat" + ao_form["email"] = "somebody@igorville.gov" + + submission = ao_form.submit() + + # A 302 indicates this page underwent a redirect. + self.assertEqual(submission.status_code, 302) + + followed_submission = submission.follow() + + # Test the returned form for data accuracy. These values should be unchanged. + new_form = followed_submission.forms[0] + self.assert_all_form_fields_have_expected_values(new_form, test_cases, test_for_disabled=True) + + # refresh domain information. Test these values in the DB. + self.domain_information.refresh_from_db() + + # All values should be unchanged. These are defined manually for code clarity. + self.assertEqual("Apple", self.domain_information.authorizing_official.first_name) + self.assertEqual("Tester", self.domain_information.authorizing_official.last_name) + self.assertEqual("CIO", self.domain_information.authorizing_official.title) + self.assertEqual("nobody@igorville.gov", self.domain_information.authorizing_official.email) + + def test_domain_edit_authorizing_official_tribal(self): + """Tests that no edit can occur when the underlying domain is tribal""" + + # Set the org type to federal + self.domain_information.organization_type = DomainInformation.OrganizationChoices.TRIBAL + self.domain_information.save() + + # Add an AO. We can do this at the model level, just not the form level. + self.domain_information.authorizing_official = Contact( + first_name="Apple", last_name="Tester", title="CIO", email="nobody@igorville.gov" + ) + self.domain_information.authorizing_official.save() + self.domain_information.save() + + ao_page = self.app.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Test if the form is populating data correctly + ao_form = ao_page.forms[0] + + test_cases = [ + ("first_name", "Apple"), + ("last_name", "Tester"), + ("title", "CIO"), + ("email", "nobody@igorville.gov"), + ] + self.assert_all_form_fields_have_expected_values(ao_form, test_cases, test_for_disabled=True) + + # Attempt to change data on each field. Because this domain is federal, + # this should not succeed. + ao_form["first_name"] = "Orange" + ao_form["last_name"] = "Smoothie" + ao_form["title"] = "Cat" + ao_form["email"] = "somebody@igorville.gov" + + submission = ao_form.submit() + + # A 302 indicates this page underwent a redirect. + self.assertEqual(submission.status_code, 302) + + followed_submission = submission.follow() + + # Test the returned form for data accuracy. These values should be unchanged. + new_form = followed_submission.forms[0] + self.assert_all_form_fields_have_expected_values(new_form, test_cases, test_for_disabled=True) + + # refresh domain information. Test these values in the DB. + self.domain_information.refresh_from_db() + + # All values should be unchanged. These are defined manually for code clarity. + self.assertEqual("Apple", self.domain_information.authorizing_official.first_name) + self.assertEqual("Tester", self.domain_information.authorizing_official.last_name) + self.assertEqual("CIO", self.domain_information.authorizing_official.title) + self.assertEqual("nobody@igorville.gov", self.domain_information.authorizing_official.email) + def test_domain_edit_authorizing_official_creates_new(self): """When editing an authorizing official for domain information and AO IS joined to another object""" @@ -1087,7 +1225,7 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Not igorville") self.assertContains(success_result_page, "Faketown") - + def test_domain_org_name_address_form_tribal(self): """ Submitting a change to organization_name is blocked for tribal domains @@ -1113,11 +1251,11 @@ class TestDomainOrganization(TestDomainOverview): # Check the value of the input field organization_name_input = form.fields["organization_name"][0] self.assertEqual(organization_name_input.value, "Town of Igorville") - + # Check if the input field is disabled self.assertTrue("disabled" in organization_name_input.attrs) self.assertEqual(organization_name_input.attrs.get("disabled"), "") - + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] org_name_page.form["organization_name"] = "Not igorville" @@ -1133,12 +1271,22 @@ class TestDomainOrganization(TestDomainOverview): self.assertContains(success_result_page, "Town of Igorville") self.assertNotContains(success_result_page, "Not igorville") + # Do another check on the form itself + form = success_result_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["organization_name"][0] + self.assertEqual(organization_name_input.value, "Town of Igorville") + + # Check if the input field is disabled + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") + # Check for the value we want to update self.assertContains(success_result_page, "Faketown") - - def test_domain_org_name_address_form_federal(self): + + def test_domain_org_name_address_form_federal_disabled(self): """ - Submitting a change to federal_agency is blocked for federal domains + Tests if the federal_agency field is readonly """ # Set the current domain to a tribal organization with a preset value. # Save first, so we can test if saving is unaffected (it should be). @@ -1155,33 +1303,48 @@ class TestDomainOrganization(TestDomainOverview): self.assertEqual(self.domain_information.organization_type, federal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) + print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field - federal_agency_input = form.fields["federal_agency"][0] - self.assertEqual(federal_agency_input.value, "AMTRAK") - - # Check if the input field is disabled - self.assertTrue("disabled" in federal_agency_input.attrs) - self.assertEqual(federal_agency_input.attrs.get("disabled"), "") - - session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + organization_name_input = form.fields["federal_agency"][0] - org_name_page.form["organization_name"] = "Not igorville" - org_name_page.form["city"] = "Faketown" + # Check if the input field is disabled. + # Webtest has some issues dealing with Selects, so we can't + # directly test the value but we can test its attributes. + # This is done in another test. + self.assertTrue("disabled" in organization_name_input.attrs) + self.assertEqual(organization_name_input.attrs.get("disabled"), "") - self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + def test_federal_agency_submit_blocked(self): + """ + Submitting a change to federal_agency is blocked for federal domains + """ + # Set the current domain to a tribal organization with a preset value. + # Save first, so we can test if saving is unaffected (it should be). + federal_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = federal_org_type + self.domain_information.save() - # Make the change. The org name should be unchanged, but city should be modifiable. - success_result_page = org_name_page.form.submit() - self.assertEqual(success_result_page.status_code, 200) + old_federal_agency_value = ("AMTRAK", "AMTRAK") + try: + # Add a federal agency. Defined as a tuple since this list may change order. + self.domain_information.federal_agency = old_federal_agency_value + self.domain_information.save() + except ValueError as err: + self.fail(f"A ValueError was caught during the test: {err}") - # Check for the old and new value - self.assertContains(success_result_page, "Town of Igorville") - self.assertNotContains(success_result_page, "Not igorville") + self.assertEqual(self.domain_information.organization_type, federal_org_type) - # Check for the value we want to update - self.assertContains(success_result_page, "Faketown") + new_value = ("Department of State", "Department of State") + self.client.post( + reverse("domain-org-name-address", kwargs={"pk": self.domain.id}), + { + "federal_agency": new_value, + }, + ) + self.assertEqual(self.domain_information.federal_agency, old_federal_agency_value) + self.assertNotEqual(self.domain_information.federal_agency, new_value) class TestDomainContactInformation(TestDomainOverview): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 07b695356..34628bc88 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -135,7 +135,7 @@ class DomainFormBaseView(DomainBaseView, FormMixin): # superclass has the redirect return super().form_invalid(form) - + def get_domain_info_from_domain(self) -> DomainInformation | None: """ Grabs the underlying domain_info object based off of self.object.name. @@ -147,7 +147,7 @@ class DomainFormBaseView(DomainBaseView, FormMixin): current_domain_info = _domain_info.get() else: logger.error("Could get domain_info. No domain info exists, or duplicates exist.") - + return current_domain_info @@ -237,7 +237,7 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): domain_info = self.get_domain_info_from_domain() invalid_fields = [DomainApplication.OrganizationChoices.FEDERAL, DomainApplication.OrganizationChoices.TRIBAL] is_federal_or_tribal = domain_info and (domain_info.organization_type in invalid_fields) - + form_kwargs["disable_fields"] = is_federal_or_tribal return form_kwargs From 707ceb286f6c97191451b28941cb3b0ec8d409a9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:53:19 -0700 Subject: [PATCH 051/125] Update _forms.scss --- src/registrar/assets/sass/_theme/_forms.scss | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/assets/sass/_theme/_forms.scss b/src/registrar/assets/sass/_theme/_forms.scss index bfd738304..86ccd101f 100644 --- a/src/registrar/assets/sass/_theme/_forms.scss +++ b/src/registrar/assets/sass/_theme/_forms.scss @@ -39,8 +39,7 @@ legend.float-left-tablet + button.float-right-tablet { } } -/* Custom style for disabled inputs */ -// TODO - UPDATE THIS! +// Custom style for disabled inputs .usa-input:disabled, .usa-select:disabled, .usa-textarea:disabled { background-color: --body-fg; color: --close-button-hover-bg; From 06f5a7f322704192b8510fb4e328c66428b94153 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:58:13 -0700 Subject: [PATCH 052/125] Update domain_org_name_address.html --- src/registrar/templates/domain_org_name_address.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/registrar/templates/domain_org_name_address.html b/src/registrar/templates/domain_org_name_address.html index 587ba4782..3a5254346 100644 --- a/src/registrar/templates/domain_org_name_address.html +++ b/src/registrar/templates/domain_org_name_address.html @@ -11,6 +11,13 @@

          The name of your organization will be publicly listed as the domain registrant.

          + {% if domain.domain_info.organization_type == 'federal' %} +

          + The federal agency for your organization can’t be updated here. + To suggest an update, email help@get.gov. +

          + {% endif %} + {% include "includes/required_fields.html" %}
          From 6e25d0e000667972c77b6630a2234d1354877afa Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:58:13 -0700 Subject: [PATCH 053/125] Change federal agency to textinput --- src/registrar/forms/domain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 5eb043742..2d67ed427 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -351,11 +351,11 @@ class DomainOrgNameAddressForm(forms.ModelForm): }, } widgets = { - # We need to set the required attributed for federal_agency and - # state/territory because for these fields we are creating an individual + # We need to set the required attributed for State/territory + # because for this fields we are creating an individual # instance of the Select. For the other fields we use the for loop to set # the class's required attribute to true. - "federal_agency": forms.Select(attrs={"required": True}, choices=DomainInformation.AGENCY_CHOICES), + "federal_agency": forms.TextInput, "organization_name": forms.TextInput, "address_line1": forms.TextInput, "address_line2": forms.TextInput, From bac44a98149d1a446895374a4a1932223061d9a5 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 6 Mar 2024 15:54:05 -0800 Subject: [PATCH 054/125] Address variable and wording feedback --- .github/workflows/daily-csv-upload.yaml | 4 ++-- .../operations/runbooks/rotate_application_secrets.md | 6 ++++-- src/Pipfile | 3 +-- src/registrar/config/settings.py | 1 + .../commands/generate_current_metadata_report.py | 4 +++- src/registrar/tests/test_emails.py | 1 + src/registrar/utility/email.py | 11 ++++++++--- 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/daily-csv-upload.yaml b/.github/workflows/daily-csv-upload.yaml index 84881398e..2a57c2083 100644 --- a/.github/workflows/daily-csv-upload.yaml +++ b/.github/workflows/daily-csv-upload.yaml @@ -31,12 +31,12 @@ jobs: cf_space: ${{ secrets.CF_REPORT_ENV }} cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full" - - name: Generate current-metadata.csv + - name: Generate and email domain-metadata-.csv uses: cloud-gov/cg-cli-tools@main with: cf_username: ${{ secrets[env.CF_USERNAME] }} cf_password: ${{ secrets[env.CF_PASSWORD] }} cf_org: cisa-dotgov cf_space: ${{ secrets.CF_REPORT_ENV }} - cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_metadata_report' --name metadata" + cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py email_current_metadata_report' --name metadata" diff --git a/docs/operations/runbooks/rotate_application_secrets.md b/docs/operations/runbooks/rotate_application_secrets.md index f7a5004ef..1094b4ff7 100644 --- a/docs/operations/runbooks/rotate_application_secrets.md +++ b/docs/operations/runbooks/rotate_application_secrets.md @@ -120,6 +120,8 @@ This is the hostname at which the registry can be found. ## SECRET_METADATA_KEY -This is in reference to the key for the metadata email that is sent daily. Reach out to product team members or leads with access to security passwords if the passcode is needed. +This is the passphrase for the zipped and encrypted metadata email that is sent out daily. Reach out to product team members or leads with access to security passwords if the passcode is needed. + +To change the password, use a password generator to generate a password, then update the user credentials per the above instructions. Be sure to update the [KBDX](https://docs.google.com/document/d/1_BbJmjYZNYLNh4jJPPnUEG9tFCzJrOc0nMrZrnSKKyw) file in Google Drive with this password change. + -To change the password, use a password generator to generate a password, then update the user credentials per the above instructions. Be sure to update the `KDBX` file in Google Drive with this password change. diff --git a/src/Pipfile b/src/Pipfile index b9c5d72d4..9208fada5 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -45,5 +45,4 @@ django-webtest = "*" types-cachetools = "*" boto3-mocking = "*" boto3-stubs = "*" -django-model2puml = "*" - +django-model2puml = "*" \ No newline at end of file diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 65b372fac..e5b97748a 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -74,6 +74,7 @@ secret_aws_s3_key_id = secret("access_key_id", None) or secret("AWS_S3_ACCESS_KE secret_aws_s3_key = secret("secret_access_key", None) or secret("AWS_S3_SECRET_ACCESS_KEY", None) secret_aws_s3_bucket_name = secret("bucket", None) or secret("AWS_S3_BUCKET_NAME", None) +# Passphrase for the encrypted metadata email secret_encrypt_metadata = secret("SECRET_ENCRYPT_METADATA", None) secret_registry_cl_id = secret("REGISTRY_CL_ID") diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 209ed5cd5..103ce0dab 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -45,7 +45,7 @@ class Command(BaseCommand): # TODO - #1317: Notify operations when auto report generation fails raise err else: - logger.info(f"Success! Created {file_name}") + logger.info(f"Success! Created {file_name} and successfully sent out an email!") def generate_current_metadata_report(self, directory, file_name, check_path): """Creates a current-metadata.csv file under the specified directory, @@ -68,6 +68,7 @@ class Command(BaseCommand): # Set zip file name current_date = datetime.now().strftime("%m%d%Y") current_filename = f"domain-metadata-{current_date}.zip" + # Pre-set zip file name encrypted_metadata_output = current_filename @@ -92,6 +93,7 @@ class Command(BaseCommand): ) def _encrypt_metadata(self, input_file, output_file, password): + """Helper function for encrypting the attachment file""" current_date = datetime.now().strftime("%m%d%Y") current_filename = f"domain-metadata-{current_date}.csv" # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster diff --git a/src/registrar/tests/test_emails.py b/src/registrar/tests/test_emails.py index 99db0d644..b11f21da6 100644 --- a/src/registrar/tests/test_emails.py +++ b/src/registrar/tests/test_emails.py @@ -211,3 +211,4 @@ class TestEmails(TestCase): self.assertIn("Content-Type: application/octet-stream", call_args["RawMessage"]["Data"]) self.assertIn("Content-Transfer-Encoding: base64", call_args["RawMessage"]["Data"]) self.assertIn("Content-Disposition: attachment;", call_args["RawMessage"]["Data"]) + self.assertNotIn("Attachment file content", call_args["RawMessage"]["Data"]) diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 35a4ecf03..91d55f361 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -20,7 +20,12 @@ class EmailSendingError(RuntimeError): def send_templated_email( - template_name: str, subject_template_name: str, to_address: str, bcc_address="", context={}, file: str = None + template_name: str, + subject_template_name: str, + to_address: str, + bcc_address="", + context={}, + attachment_file: str = None, ): """Send an email built from a template to one email address. @@ -51,7 +56,7 @@ def send_templated_email( destination["BccAddresses"] = [bcc_address] try: - if file is None: + if attachment_file is None: ses_client.send_email( FromEmailAddress=settings.DEFAULT_FROM_EMAIL, Destination=destination, @@ -71,7 +76,7 @@ def send_templated_email( config=settings.BOTO_CONFIG, ) response = send_email_with_attachment( - settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, file, ses_client + settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, attachment_file, ses_client ) # TODO: Remove this print statement when ready to merge, # leaving rn for getting error codes in case From 370b2ddda068bf77971f7eccbfa40ec4e0a0be7a Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 6 Mar 2024 16:05:41 -0800 Subject: [PATCH 055/125] Fix variable name --- .../management/commands/generate_current_metadata_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index 103ce0dab..da5aafb91 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -89,7 +89,7 @@ class Command(BaseCommand): # to_address=settings.DEFAULT_FROM_EMAIL, # TODO: Uncomment this when ready to merge to_address="rebecca.hsieh@truss.works ", context={"current_date_str": current_date_str}, - file=encrypted_metadata_in_bytes, + attachment_file=encrypted_metadata_in_bytes, ) def _encrypt_metadata(self, input_file, output_file, password): From 8438067d79a4a864994a70d9ac12522df58d2bbb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 08:46:35 -0700 Subject: [PATCH 056/125] Update domain.py --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 2d67ed427..e01b2cd03 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -351,7 +351,7 @@ class DomainOrgNameAddressForm(forms.ModelForm): }, } widgets = { - # We need to set the required attributed for State/territory + # We need to set the required attributed for State/territory # because for this fields we are creating an individual # instance of the Select. For the other fields we use the for loop to set # the class's required attribute to true. From 52f299d06e304cca79026037facee6794d5f1bb7 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 7 Mar 2024 12:35:10 -0500 Subject: [PATCH 057/125] initial form changes - wip --- src/registrar/admin.py | 58 +++++++--------------- src/registrar/models/domain_application.py | 4 +- src/registrar/models/domain_information.py | 2 +- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 92e477667..0ea791181 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -777,18 +777,21 @@ class DomainInformationAdmin(ListHeaderAdmin): search_help_text = "Search by domain." fieldsets = [ - (None, {"fields": ["creator", "domain_application", "notes"]}), + (None, {"fields": ["creator", "submitter", "domain_application", "notes"]}), + (".gov domain", {"fields": ["domain"]}), + ("Contacts", {"fields": ["authorizing_official", "other_contacts", "no_other_contacts_rationale"]}), + ("Background info", {"fields": ["anything_else"]}), ( "Type of organization", { "fields": [ "organization_type", + "is_election_board", + "federal_type", + "federal_agency", + "tribe_name", "federally_recognized_tribe", "state_recognized_tribe", - "tribe_name", - "federal_agency", - "federal_type", - "is_election_board", "about_your_organization", ] }, @@ -798,28 +801,15 @@ class DomainInformationAdmin(ListHeaderAdmin): { "fields": [ "organization_name", + "state_territory", "address_line1", "address_line2", "city", - "state_territory", "zipcode", "urbanization", ] }, ), - ("Authorizing official", {"fields": ["authorizing_official"]}), - (".gov domain", {"fields": ["domain"]}), - ("Your contact information", {"fields": ["submitter"]}), - ("Other employees from your organization?", {"fields": ["other_contacts"]}), - ( - "No other employees from your organization?", - {"fields": ["no_other_contacts_rationale"]}, - ), - ("Anything else?", {"fields": ["anything_else"]}), - ( - "Requirements for operating a .gov domain", - {"fields": ["is_policy_acknowledged"]}, - ), ] # Read only that we'll leverage for CISA Analysts @@ -979,18 +969,21 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "approved_domain", "notes"]}), + (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "submitter", "approved_domain", "notes"]}), + (".gov domain", {"fields": ["requested_domain", "alternative_domains"]}), + ("Contacts", {"fields": ["authorizing_official", "other_contacts", "no_other_contacts_rationale"]}), + ("Background info", {"fields": ["purpose", "anything_else", "current_websites"]}), ( "Type of organization", { "fields": [ "organization_type", + "is_election_board", + "federal_type", + "federal_agency", + "tribe_name", "federally_recognized_tribe", "state_recognized_tribe", - "tribe_name", - "federal_agency", - "federal_type", - "is_election_board", "about_your_organization", ] }, @@ -1000,30 +993,15 @@ class DomainApplicationAdmin(ListHeaderAdmin): { "fields": [ "organization_name", + "state_territory", "address_line1", "address_line2", "city", - "state_territory", "zipcode", "urbanization", ] }, ), - ("Authorizing official", {"fields": ["authorizing_official"]}), - ("Current websites", {"fields": ["current_websites"]}), - (".gov domain", {"fields": ["requested_domain", "alternative_domains"]}), - ("Purpose of your domain", {"fields": ["purpose"]}), - ("Your contact information", {"fields": ["submitter"]}), - ("Other employees from your organization?", {"fields": ["other_contacts"]}), - ( - "No other employees from your organization?", - {"fields": ["no_other_contacts_rationale"]}, - ), - ("Anything else?", {"fields": ["anything_else"]}), - ( - "Requirements for operating a .gov domain", - {"fields": ["is_policy_acknowledged"]}, - ), ] # Read only that we'll leverage for CISA Analysts diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index 8c417b51a..90627a63b 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -504,7 +504,7 @@ class DomainApplication(TimeStampedModel): "registrar.Website", blank=True, related_name="current+", - verbose_name="websites", + verbose_name="Current websites", ) approved_domain = models.OneToOneField( @@ -550,7 +550,7 @@ class DomainApplication(TimeStampedModel): "registrar.Contact", blank=True, related_name="contact_applications", - verbose_name="contacts", + verbose_name="Other employees", ) no_other_contacts_rationale = models.TextField( diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 861171c5c..5e17cfd2c 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -183,7 +183,7 @@ class DomainInformation(TimeStampedModel): "registrar.Contact", blank=True, related_name="contact_applications_information", - verbose_name="contacts", + verbose_name="Other employees", ) no_other_contacts_rationale = models.TextField( From aa26f782d039920f4937c3b818962a2d1b5d2490 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 7 Mar 2024 12:43:40 -0500 Subject: [PATCH 058/125] lint and migrations --- src/registrar/admin.py | 15 +++++++- ...inapplication_current_websites_and_more.py | 37 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 0ea791181..f03fe6713 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -969,7 +969,20 @@ class DomainApplicationAdmin(ListHeaderAdmin): search_help_text = "Search by domain or submitter." fieldsets = [ - (None, {"fields": ["status", "rejection_reason", "investigator", "creator", "submitter", "approved_domain", "notes"]}), + ( + None, + { + "fields": [ + "status", + "rejection_reason", + "investigator", + "creator", + "submitter", + "approved_domain", + "notes", + ] + }, + ), (".gov domain", {"fields": ["requested_domain", "alternative_domains"]}), ("Contacts", {"fields": ["authorizing_official", "other_contacts", "no_other_contacts_rationale"]}), ("Background info", {"fields": ["purpose", "anything_else", "current_websites"]}), diff --git a/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py b/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py new file mode 100644 index 000000000..cefaba27c --- /dev/null +++ b/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.10 on 2024-03-07 17:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0072_alter_publiccontact_fax_alter_publiccontact_voice"), + ] + + operations = [ + migrations.AlterField( + model_name="domainapplication", + name="current_websites", + field=models.ManyToManyField( + blank=True, related_name="current+", to="registrar.website", verbose_name="Current websites" + ), + ), + migrations.AlterField( + model_name="domainapplication", + name="other_contacts", + field=models.ManyToManyField( + blank=True, related_name="contact_applications", to="registrar.contact", verbose_name="Other employees" + ), + ), + migrations.AlterField( + model_name="domaininformation", + name="other_contacts", + field=models.ManyToManyField( + blank=True, + related_name="contact_applications_information", + to="registrar.contact", + verbose_name="Other employees", + ), + ), + ] From 1e90c3121ba0e493938345c6667a93688c4b9383 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 7 Mar 2024 14:17:41 -0500 Subject: [PATCH 059/125] changed font size of h2 --- src/registrar/assets/sass/_theme/_admin.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index b57c6a015..e53dc8b4b 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -141,6 +141,10 @@ h1, h2, h3, font-weight: font-weight('bold'); } +#content h2 { + font-size: 1.3rem; +} + .module h3 { padding: 0; color: var(--link-fg); From ed7d1d8c4fed4df2c89c7582e498cfba0742394c Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 7 Mar 2024 13:21:52 -0800 Subject: [PATCH 060/125] Fix script name --- .../management/commands/generate_current_metadata_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/generate_current_metadata_report.py index da5aafb91..cdda32002 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/generate_current_metadata_report.py @@ -40,14 +40,14 @@ class Command(BaseCommand): logger.info("Generating report...") try: - self.generate_current_metadata_report(directory, file_name, check_path) + self.email_current_metadata_report(directory, file_name, check_path) except Exception as err: # TODO - #1317: Notify operations when auto report generation fails raise err else: logger.info(f"Success! Created {file_name} and successfully sent out an email!") - def generate_current_metadata_report(self, directory, file_name, check_path): + def email_current_metadata_report(self, directory, file_name, check_path): """Creates a current-metadata.csv file under the specified directory, then uploads it to a AWS S3 bucket. This is done for resiliency reasons in the event our application goes down and/or the email From 7e6cf54781aa89568d4374830b9f9055e187d4f2 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 7 Mar 2024 13:34:40 -0800 Subject: [PATCH 061/125] Fix renaming for yaml upload --- .github/workflows/daily-csv-upload.yaml | 2 +- ..._metadata_report.py => email_current_metadata_report.py} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/registrar/management/commands/{generate_current_metadata_report.py => email_current_metadata_report.py} (95%) diff --git a/.github/workflows/daily-csv-upload.yaml b/.github/workflows/daily-csv-upload.yaml index 2a57c2083..9cacfc3bf 100644 --- a/.github/workflows/daily-csv-upload.yaml +++ b/.github/workflows/daily-csv-upload.yaml @@ -31,7 +31,7 @@ jobs: cf_space: ${{ secrets.CF_REPORT_ENV }} cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full" - - name: Generate and email domain-metadata-.csv + - name: Generate and email domain-metadata.csv uses: cloud-gov/cg-cli-tools@main with: cf_username: ${{ secrets[env.CF_USERNAME] }} diff --git a/src/registrar/management/commands/generate_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py similarity index 95% rename from src/registrar/management/commands/generate_current_metadata_report.py rename to src/registrar/management/commands/email_current_metadata_report.py index cdda32002..4300bf227 100644 --- a/src/registrar/management/commands/generate_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): help = ( - "Generates and uploads a current-metadata.csv file to our S3 bucket " + "Generates and uploads a domain-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." ) @@ -32,8 +32,8 @@ class Command(BaseCommand): ) def handle(self, **options): - """Grabs the directory then creates current-metadata.csv in that directory""" - file_name = "current-metadata.csv" + """Grabs the directory then creates domain-metadata.csv in that directory""" + file_name = "domain-metadata.csv" # Ensures a slash is added directory = os.path.join(options.get("directory"), "") check_path = options.get("checkpath") From 4e67733861a778fb783858cd88d4caadd8656b68 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:44:13 -0700 Subject: [PATCH 062/125] Update test_views_domain.py --- src/registrar/tests/test_views_domain.py | 51 +++++++++++++++++------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 2a861a878..6da1ce66e 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1245,7 +1245,6 @@ class TestDomainOrganization(TestDomainOverview): self.assertEqual(self.domain_information.organization_type, tribal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) - print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field @@ -1284,38 +1283,62 @@ class TestDomainOrganization(TestDomainOverview): # Check for the value we want to update self.assertContains(success_result_page, "Faketown") - def test_domain_org_name_address_form_federal_disabled(self): + def test_domain_org_name_address_form_federal(self): """ - Tests if the federal_agency field is readonly + Submitting a change to federal_agency is blocked for federal domains """ # Set the current domain to a tribal organization with a preset value. # Save first, so we can test if saving is unaffected (it should be). - federal_org_type = DomainInformation.OrganizationChoices.FEDERAL - self.domain_information.organization_type = federal_org_type + fed_org_type = DomainInformation.OrganizationChoices.FEDERAL + self.domain_information.organization_type = fed_org_type self.domain_information.save() try: - # Add a federal agency. Defined as a tuple since this list may change order. - self.domain_information.federal_agency = ("AMTRAK", "AMTRAK") + self.domain_information.federal_agency = "AMTRAK" self.domain_information.save() except ValueError as err: self.fail(f"A ValueError was caught during the test: {err}") - self.assertEqual(self.domain_information.organization_type, federal_org_type) + self.assertEqual(self.domain_information.organization_type, tribal_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) - print(f"what is the org name page? {org_name_page}") form = org_name_page.forms[0] # Check the value of the input field - organization_name_input = form.fields["federal_agency"][0] + agency_input = form.fields["federal_agency"][0] + self.assertEqual(agency_input.value, "AMTRAK") - # Check if the input field is disabled. - # Webtest has some issues dealing with Selects, so we can't - # directly test the value but we can test its attributes. - # This is done in another test. + # Check if the input field is disabled + self.assertTrue("disabled" in agency_input.attrs) + self.assertEqual(agency_input.attrs.get("disabled"), "") + + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + org_name_page.form["federal_agency"] = "Department of State" + org_name_page.form["city"] = "Faketown" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Make the change. The agency should be unchanged, but city should be modifiable. + success_result_page = org_name_page.form.submit() + self.assertEqual(success_result_page.status_code, 200) + + # Check for the old and new value + self.assertContains(success_result_page, "AMTRAK") + self.assertNotContains(success_result_page, "Department of State") + + # Do another check on the form itself + form = success_result_page.forms[0] + # Check the value of the input field + organization_name_input = form.fields["federal_agency"][0] + self.assertEqual(organization_name_input.value, "AMTRAK") + + # Check if the input field is disabled self.assertTrue("disabled" in organization_name_input.attrs) self.assertEqual(organization_name_input.attrs.get("disabled"), "") + # Check for the value we want to update + self.assertContains(success_result_page, "Faketown") + def test_federal_agency_submit_blocked(self): """ Submitting a change to federal_agency is blocked for federal domains From e900007c957c8b874865599c750fc7aa38227a81 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:59:14 -0700 Subject: [PATCH 063/125] Update test_views_domain.py --- src/registrar/tests/test_views_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 6da1ce66e..6aede926f 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1298,7 +1298,7 @@ class TestDomainOrganization(TestDomainOverview): except ValueError as err: self.fail(f"A ValueError was caught during the test: {err}") - self.assertEqual(self.domain_information.organization_type, tribal_org_type) + self.assertEqual(self.domain_information.organization_type, fed_org_type) org_name_page = self.app.get(reverse("domain-org-name-address", kwargs={"pk": self.domain.id})) From 5f60134d1dbb1f14028537b2d1a5e936a8991efc Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 7 Mar 2024 19:20:08 -0500 Subject: [PATCH 064/125] updated style on h2 --- src/registrar/assets/sass/_theme/_admin.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index e53dc8b4b..fa3efb9a2 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -141,7 +141,7 @@ h1, h2, h3, font-weight: font-weight('bold'); } -#content h2 { +div#content > h2 { font-size: 1.3rem; } From 33b47d27d17a75d8f54281302b46f36bcab3c995 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:03:10 -0800 Subject: [PATCH 065/125] handle none --- src/registrar/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 5b8d67983..68f27d15c 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,7 +376,10 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time = f"{avg_approval_time.days} days" + + avg_approval_time="No approvals to use" + if avg_approval_time is not None: + avg_approval_time = f"{avg_approval_time.days} days" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") From d1bac52aa61a1279788fc9430d593a4ec2968f1b Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:08:24 -0800 Subject: [PATCH 066/125] moved code line --- src/registrar/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 68f27d15c..4b841ab12 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -377,10 +377,11 @@ def analytics(request): ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - avg_approval_time="No approvals to use" + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" - + else: + avg_approval_time="No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From bd352443964ebf6aae03a67c5ace122df07b03c7 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Thu, 7 Mar 2024 20:33:22 -0800 Subject: [PATCH 067/125] lint --- src/registrar/admin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 4b841ab12..59aa2ace8 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -376,12 +376,11 @@ def analytics(request): approval_time=F("approved_domain__created_at") - F("submission_date") ).aggregate(Avg("approval_time"))["approval_time__avg"] # Format the timedelta to display only days - - + if avg_approval_time is not None: avg_approval_time = f"{avg_approval_time.days} days" else: - avg_approval_time="No approvals to use" + avg_approval_time = "No approvals to use" # The start and end dates are passed as url params start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") From 47147abf72ddf3c9b6ca915d9898eb3c95b330d6 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 8 Mar 2024 10:55:40 -0800 Subject: [PATCH 068/125] Update from created at to submission date --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 92e477667..4b85b3f3a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -942,7 +942,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): "custom_election_board", "city", "state_territory", - "created_at", + "submission_date", #this is the right change "submitter", "investigator", ] From 139e7f11830f4a6b1fc8ee6be861cde205bb0b82 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 8 Mar 2024 14:45:02 -0500 Subject: [PATCH 069/125] remove .gov domain from domain form --- src/registrar/admin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index f03fe6713..fb2ad50af 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,5 +1,6 @@ from datetime import date import logging +import copy from django import forms from django.db.models.functions import Concat, Coalesce @@ -1185,7 +1186,13 @@ class DomainInformationInline(admin.StackedInline): model = models.DomainInformation - fieldsets = DomainInformationAdmin.fieldsets + fieldsets = copy.deepcopy(DomainInformationAdmin.fieldsets) + # remove .gov domain from fieldset + for index, (title, _) in enumerate(fieldsets): + if title == ".gov domain": + del fieldsets[index] + break + analyst_readonly_fields = DomainInformationAdmin.analyst_readonly_fields # For each filter_horizontal, init in admin js extendFilterHorizontalWidgets # to activate the edit/delete/view buttons From feafc74d05aebc0969676cc4a6bc0a37703ec1bb Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 8 Mar 2024 11:49:52 -0800 Subject: [PATCH 070/125] Remove comment --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 4b85b3f3a..5c24aaf79 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -942,7 +942,7 @@ class DomainApplicationAdmin(ListHeaderAdmin): "custom_election_board", "city", "state_territory", - "submission_date", #this is the right change + "submission_date", "submitter", "investigator", ] From 29e3275a87a18eac20449565672fbbae0c358f56 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 8 Mar 2024 14:52:01 -0500 Subject: [PATCH 071/125] satisfied linter --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index fb2ad50af..eb88473f8 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1188,7 +1188,7 @@ class DomainInformationInline(admin.StackedInline): fieldsets = copy.deepcopy(DomainInformationAdmin.fieldsets) # remove .gov domain from fieldset - for index, (title, _) in enumerate(fieldsets): + for index, (title, f) in enumerate(fieldsets): if title == ".gov domain": del fieldsets[index] break From 454cac951ab6cf9f5cb51103092c3bb849f54408 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:41:40 -0700 Subject: [PATCH 072/125] Bug fix for PR --- src/registrar/assets/js/get-gov-admin.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 4ed1a0d28..8170e4bd0 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -61,18 +61,19 @@ function openInNewTab(el, removeAttribute = false){ * This intentionally does not interact with createPhantomModalFormButtons() */ (function (){ - function displayModalOnDropdownClick(linkClickedDisplaysModal, statusDropdown, cancelButton, valueToCheck){ + function displayModalOnDropdownClick(linkClickedDisplaysModal, statusDropdown, actionButton, valueToCheck){ // If these exist all at the same time, we're on the right page if (linkClickedDisplaysModal && statusDropdown && statusDropdown.value){ + + // Set the previous value in the event the user cancels. + let previousValue = statusDropdown.value; + if (actionButton){ - if (cancelButton){ - // Store the previous value in the event the user cancels. - // We only need to do this if cancel button is specified. - let previousValue = statusDropdown.value; - cancelButton.addEventListener('click', function() { + // Otherwise, if the confirmation buttion is pressed, set it to that + actionButton.addEventListener('click', function() { // Revert the dropdown to its previous value - statusDropdown.value = previousValue; + statusDropdown.value = valueToCheck; }); }else { console.log("displayModalOnDropdownClick() -> Cancel button was null") @@ -82,6 +83,10 @@ function openInNewTab(el, removeAttribute = false){ statusDropdown.addEventListener('change', function() { // Check if "Ineligible" is selected if (this.value && this.value.toLowerCase() === valueToCheck) { + // Set the old value in the event the user cancels, + // or otherwise exists the dropdown. + statusDropdown.value = previousValue + // Display the modal. linkClickedDisplaysModal.click() } @@ -98,9 +103,9 @@ function openInNewTab(el, removeAttribute = false){ // Because the modal button does not have the class "dja-form-placeholder", // it will not be affected by the createPhantomModalFormButtons() function. - let cancelButton = document.querySelector('button[name="_cancel_application_ineligible"]'); + let actionButton = document.querySelector('button[name="_set_application_ineligible"]'); let valueToCheck = "ineligible" - displayModalOnDropdownClick(modalButton, statusDropdown, cancelButton, valueToCheck); + displayModalOnDropdownClick(modalButton, statusDropdown, actionButton, valueToCheck); } hookModalToIneligibleStatus() From d75cbf50e3268d4a69ec57ddc95b1c8b19bafd54 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 9 Mar 2024 06:41:10 -0500 Subject: [PATCH 073/125] merge changes --- src/registrar/admin.py | 2 +- src/registrar/assets/sass/_theme/_admin.scss | 4 ++ ...maininformation_other_contacts_and_more.py | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/registrar/migrations/0075_alter_domaininformation_other_contacts_and_more.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index e877d8ef8..6e4e7f50f 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -777,7 +777,7 @@ class DomainInformationAdmin(ListHeaderAdmin): search_help_text = "Search by domain." fieldsets = [ - (None, {"fields": ["creator", "submitter", "domain_application", "notes"]}), + (None, {"fields": ["creator", "submitter", "domain_request", "notes"]}), (".gov domain", {"fields": ["domain"]}), ("Contacts", {"fields": ["authorizing_official", "other_contacts", "no_other_contacts_rationale"]}), ("Background info", {"fields": ["anything_else"]}), diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 9c0d7517c..aba2ad9a3 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -143,6 +143,10 @@ h1, h2, h3, font-weight: font-weight('bold'); } +div#content > h2 { + font-size: 1.3rem; +} + .module h3 { padding: 0; color: var(--link-fg); diff --git a/src/registrar/migrations/0075_alter_domaininformation_other_contacts_and_more.py b/src/registrar/migrations/0075_alter_domaininformation_other_contacts_and_more.py new file mode 100644 index 000000000..53ad96ca4 --- /dev/null +++ b/src/registrar/migrations/0075_alter_domaininformation_other_contacts_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 4.2.10 on 2024-03-09 11:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0074_create_groups_v08"), + ] + + operations = [ + migrations.AlterField( + model_name="domaininformation", + name="other_contacts", + field=models.ManyToManyField( + blank=True, + related_name="contact_domain_requests_information", + to="registrar.contact", + verbose_name="Other employees", + ), + ), + migrations.AlterField( + model_name="domainrequest", + name="current_websites", + field=models.ManyToManyField( + blank=True, related_name="current+", to="registrar.website", verbose_name="Current websites" + ), + ), + migrations.AlterField( + model_name="domainrequest", + name="other_contacts", + field=models.ManyToManyField( + blank=True, + related_name="contact_domain_requests", + to="registrar.contact", + verbose_name="Other employees", + ), + ), + ] From b27318e9755990f48ccef75273200467e48753e6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 9 Mar 2024 06:45:30 -0500 Subject: [PATCH 074/125] merge migrations --- ...inapplication_current_websites_and_more.py | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py diff --git a/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py b/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py deleted file mode 100644 index cefaba27c..000000000 --- a/src/registrar/migrations/0073_alter_domainapplication_current_websites_and_more.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 4.2.10 on 2024-03-07 17:42 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("registrar", "0072_alter_publiccontact_fax_alter_publiccontact_voice"), - ] - - operations = [ - migrations.AlterField( - model_name="domainapplication", - name="current_websites", - field=models.ManyToManyField( - blank=True, related_name="current+", to="registrar.website", verbose_name="Current websites" - ), - ), - migrations.AlterField( - model_name="domainapplication", - name="other_contacts", - field=models.ManyToManyField( - blank=True, related_name="contact_applications", to="registrar.contact", verbose_name="Other employees" - ), - ), - migrations.AlterField( - model_name="domaininformation", - name="other_contacts", - field=models.ManyToManyField( - blank=True, - related_name="contact_applications_information", - to="registrar.contact", - verbose_name="Other employees", - ), - ), - ] From ee60be8b10845519ad88bc0f4147ca9eac8a5c27 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:21:08 -0600 Subject: [PATCH 075/125] Add semaphore + add error handling for .close --- src/epplibwrapper/client.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index a7856298b..823c26288 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -1,6 +1,7 @@ """Provide a wrapper around epplib to handle authentication and errors.""" import logging +from gevent.lock import BoundedSemaphore try: from epplib.client import Client @@ -52,6 +53,9 @@ class EPPLibWrapper: "urn:ietf:params:xml:ns:contact-1.0", ], ) + # We should only ever have one active connection at a time, + # given that + self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() except Exception: @@ -91,12 +95,23 @@ class EPPLibWrapper: raise RegistryError(message) from err def _disconnect(self) -> None: - """Close the connection.""" + """Close the connection. Sends a logout command and closes the connection.""" + self._send_logout_command() + self._close_client() + + def _send_logout_command(self): + """Sends a logout command to epp""" try: self._client.send(commands.Logout()) # type: ignore - self._client.close() # type: ignore - except Exception: - logger.warning("Connection to registry was not cleanly closed.") + except Exception as err: + logger.warning(f"Logout command not sent successfully: {err}") + + def _close_client(self): + """Closes an active client connection""" + try: + self._client.close() + except Exception as err: + logger.warning(f"Connection to registry was not cleanly closed: {err}") def _send(self, command): """Helper function used by `send`.""" @@ -146,6 +161,8 @@ class EPPLibWrapper: cmd_type = command.__class__.__name__ if not cleaned: raise ValueError("Please sanitize user input before sending it.") + + self.connection_lock.acquire() try: return self._send(command) except RegistryError as err: @@ -161,6 +178,8 @@ class EPPLibWrapper: return self._retry(command) else: raise err + finally: + self.connection_lock.release() try: From 3b9f68ac4b58bb1f7b04f797d9eb2b8ddd3b6b4c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:59:55 -0600 Subject: [PATCH 076/125] linting --- src/epplibwrapper/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 823c26288..5006708d6 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -54,7 +54,7 @@ class EPPLibWrapper: ], ) # We should only ever have one active connection at a time, - # given that + # given that self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() @@ -105,7 +105,7 @@ class EPPLibWrapper: self._client.send(commands.Logout()) # type: ignore except Exception as err: logger.warning(f"Logout command not sent successfully: {err}") - + def _close_client(self): """Closes an active client connection""" try: @@ -161,7 +161,7 @@ class EPPLibWrapper: cmd_type = command.__class__.__name__ if not cleaned: raise ValueError("Please sanitize user input before sending it.") - + self.connection_lock.acquire() try: return self._send(command) From 6992f1489b30c087118576b61dadd9eff0e88b1d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 15:01:01 -0600 Subject: [PATCH 077/125] Fix comment --- src/epplibwrapper/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 5006708d6..b346563d2 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -53,8 +53,7 @@ class EPPLibWrapper: "urn:ietf:params:xml:ns:contact-1.0", ], ) - # We should only ever have one active connection at a time, - # given that + # We should only ever have one active connection at a time self.connection_lock = BoundedSemaphore(1) try: self._initialize_client() From f31d6ee0735e1014d3dd7c7e553c213ff48fac7d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:44:56 -0600 Subject: [PATCH 078/125] Update test_admin.py --- src/registrar/tests/test_admin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index fb7e7af5e..8d983dcb7 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -1224,7 +1224,7 @@ class TestDomainRequestAdmin(MockEppLib): EMAIL = "mayor@igorville.gov" User.objects.filter(email=EMAIL).delete() - # Create a sample application + # Create a sample domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW) p = "userpass" @@ -1261,7 +1261,7 @@ class TestDomainRequestAdmin(MockEppLib): # Test that approved domain exists and equals requested domain self.assertEqual(domain_request.creator.status, "restricted") - # 'Get' to the application again + # 'Get' to the domain request again response = self.client.get( "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True, @@ -1282,7 +1282,7 @@ class TestDomainRequestAdmin(MockEppLib): EMAIL = "mayor@igorville.gov" User.objects.filter(email=EMAIL).delete() - # Create a sample application + # Create a sample domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW) p = "userpass" @@ -1310,7 +1310,7 @@ class TestDomainRequestAdmin(MockEppLib): "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True ) with boto3_mocking.clients.handler_for("sesv2", self.mock_client): - # Modify the application's property + # Modify the domain request's property domain_request.status = DomainRequest.DomainRequestStatus.INELIGIBLE # Use the model admin's save_model method @@ -1319,7 +1319,7 @@ class TestDomainRequestAdmin(MockEppLib): # Test that approved domain exists and equals requested domain self.assertEqual(domain_request.creator.status, "restricted") - # 'Get' to the application again + # 'Get' to the domain request again response = self.client.get( "/admin/registrar/domainrequest/{}/change/".format(domain_request.pk), follow=True, From 240c5c8e538984ae60d89d3e6c109035495f1686 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Tue, 12 Mar 2024 16:21:17 -0700 Subject: [PATCH 079/125] added bob and meoward --- .github/workflows/migrate.yaml | 2 + .github/workflows/reset-db.yaml | 2 + ops/manifests/manifest-bob.yaml | 32 +++++++++++++ ops/manifests/manifest-meoward.yaml | 32 +++++++++++++ ops/scripts/create_dev_sandbox.sh | 72 ++++++++++++++--------------- src/registrar/config/settings.py | 2 + 6 files changed, 106 insertions(+), 36 deletions(-) create mode 100644 ops/manifests/manifest-bob.yaml create mode 100644 ops/manifests/manifest-meoward.yaml diff --git a/.github/workflows/migrate.yaml b/.github/workflows/migrate.yaml index 2033ee51c..825ab04d7 100644 --- a/.github/workflows/migrate.yaml +++ b/.github/workflows/migrate.yaml @@ -16,6 +16,8 @@ on: - stable - staging - development + - bob + - meoward - backup - ky - es diff --git a/.github/workflows/reset-db.yaml b/.github/workflows/reset-db.yaml index f8730c865..05eb963c3 100644 --- a/.github/workflows/reset-db.yaml +++ b/.github/workflows/reset-db.yaml @@ -16,6 +16,8 @@ on: options: - staging - development + - bob + - meoward - backup - ky - es diff --git a/ops/manifests/manifest-bob.yaml b/ops/manifests/manifest-bob.yaml new file mode 100644 index 000000000..f39d9e145 --- /dev/null +++ b/ops/manifests/manifest-bob.yaml @@ -0,0 +1,32 @@ +--- +applications: +- name: getgov-bob + buildpacks: + - python_buildpack + path: ../../src + instances: 1 + memory: 512M + stack: cflinuxfs4 + timeout: 180 + command: ./run.sh + health-check-type: http + health-check-http-endpoint: /health + health-check-invocation-timeout: 40 + env: + # Send stdout and stderr straight to the terminal without buffering + PYTHONUNBUFFERED: yup + # Tell Django where to find its configuration + DJANGO_SETTINGS_MODULE: registrar.config.settings + # Tell Django where it is being hosted + DJANGO_BASE_URL: https://getgov-bob.app.cloud.gov + # Tell Django how much stuff to log + DJANGO_LOG_LEVEL: INFO + # default public site location + GETGOV_PUBLIC_SITE_URL: https://get.gov + # Flag to disable/enable features in prod environments + IS_PRODUCTION: False + routes: + - route: getgov-bob.app.cloud.gov + services: + - getgov-credentials + - getgov-bob-database diff --git a/ops/manifests/manifest-meoward.yaml b/ops/manifests/manifest-meoward.yaml new file mode 100644 index 000000000..c47d9529d --- /dev/null +++ b/ops/manifests/manifest-meoward.yaml @@ -0,0 +1,32 @@ +--- +applications: +- name: getgov-meoward + buildpacks: + - python_buildpack + path: ../../src + instances: 1 + memory: 512M + stack: cflinuxfs4 + timeout: 180 + command: ./run.sh + health-check-type: http + health-check-http-endpoint: /health + health-check-invocation-timeout: 40 + env: + # Send stdout and stderr straight to the terminal without buffering + PYTHONUNBUFFERED: yup + # Tell Django where to find its configuration + DJANGO_SETTINGS_MODULE: registrar.config.settings + # Tell Django where it is being hosted + DJANGO_BASE_URL: https://getgov-meoward.app.cloud.gov + # Tell Django how much stuff to log + DJANGO_LOG_LEVEL: INFO + # default public site location + GETGOV_PUBLIC_SITE_URL: https://get.gov + # Flag to disable/enable features in prod environments + IS_PRODUCTION: False + routes: + - route: getgov-meoward.app.cloud.gov + services: + - getgov-credentials + - getgov-meoward-database diff --git a/ops/scripts/create_dev_sandbox.sh b/ops/scripts/create_dev_sandbox.sh index 676fcf7ae..975c7d997 100755 --- a/ops/scripts/create_dev_sandbox.sh +++ b/ops/scripts/create_dev_sandbox.sh @@ -7,51 +7,51 @@ if [ -z "$1" ]; then exit 1 fi -if [ ! $(command -v gh) ] || [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; then - echo "jq, cf, and gh packages must be installed. Please install via your preferred manager." - exit 1 -fi +# if [ ! $(command -v gh) ] || [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; then +# echo "jq, cf, and gh packages must be installed. Please install via your preferred manager." +# exit 1 +# fi upcase_name=$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]') -read -p "Are you on a new branch? We will have to commit this work. (y/n) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]] -then - git checkout -b new-dev-sandbox-$1 -fi +# read -p "Are you on a new branch? We will have to commit this work. (y/n) " -n 1 -r +# echo +# if [[ ! $REPLY =~ ^[Yy]$ ]] +# then +# git checkout -b new-dev-sandbox-$1 +# fi -cf target -o cisa-dotgov +# cf target -o cisa-dotgov -read -p "Are you logged in to the cisa-dotgov CF org above? (y/n) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]] -then - cf login -a https://api.fr.cloud.gov --sso -fi +# read -p "Are you logged in to the cisa-dotgov CF org above? (y/n) " -n 1 -r +# echo +# if [[ ! $REPLY =~ ^[Yy]$ ]] +# then +# cf login -a https://api.fr.cloud.gov --sso +# fi -gh auth status -read -p "Are you logged into a Github account with access to cisagov/getgov? (y/n) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]] -then - gh auth login -fi +# gh auth status +# read -p "Are you logged into a Github account with access to cisagov/getgov? (y/n) " -n 1 -r +# echo +# if [[ ! $REPLY =~ ^[Yy]$ ]] +# then +# gh auth login +# fi -echo "Creating manifest for $1..." -cp ops/scripts/manifest-sandbox-template.yaml ops/manifests/manifest-$1.yaml -sed -i '' "s/ENVIRONMENT/$1/" "ops/manifests/manifest-$1.yaml" +# echo "Creating manifest for $1..." +# cp ops/scripts/manifest-sandbox-template.yaml ops/manifests/manifest-$1.yaml +# sed -i '' "s/ENVIRONMENT/$1/" "ops/manifests/manifest-$1.yaml" -echo "Adding new environment to settings.py..." -sed -i '' '/getgov-development.app.cloud.gov/ {a\ - '\"getgov-$1.app.cloud.gov\"', -}' src/registrar/config/settings.py +# echo "Adding new environment to settings.py..." +# sed -i '' '/getgov-development.app.cloud.gov/ {a\ +# '\"getgov-$1.app.cloud.gov\"', +# }' src/registrar/config/settings.py -echo "Creating new cloud.gov space for $1..." -cf create-space $1 -cf target -o "cisa-dotgov" -s $1 -cf bind-security-group public_networks_egress cisa-dotgov --space $1 -cf bind-security-group trusted_local_networks_egress cisa-dotgov --space $1 +# echo "Creating new cloud.gov space for $1..." +# cf create-space $1 +# cf target -o "cisa-dotgov" -s $1 +# cf bind-security-group public_networks_egress cisa-dotgov --space $1 +# cf bind-security-group trusted_local_networks_egress cisa-dotgov --space $1 echo "Creating new cloud.gov DB for $1. This usually takes about 5 minutes..." cf create-service aws-rds micro-psql getgov-$1-database diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 15799f91b..1ce01ac1f 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -635,6 +635,8 @@ ALLOWED_HOSTS = [ "getgov-stable.app.cloud.gov", "getgov-staging.app.cloud.gov", "getgov-development.app.cloud.gov", + "getgov-bob.app.cloud.gov", + "getgov-meoward.app.cloud.gov", "getgov-backup.app.cloud.gov", "getgov-ky.app.cloud.gov", "getgov-es.app.cloud.gov", From bc2843bfe7d7032f931550520f985ab93772827c Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Tue, 12 Mar 2024 16:24:10 -0700 Subject: [PATCH 080/125] undo code comments --- ops/scripts/create_dev_sandbox.sh | 72 +++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ops/scripts/create_dev_sandbox.sh b/ops/scripts/create_dev_sandbox.sh index 975c7d997..676fcf7ae 100755 --- a/ops/scripts/create_dev_sandbox.sh +++ b/ops/scripts/create_dev_sandbox.sh @@ -7,51 +7,51 @@ if [ -z "$1" ]; then exit 1 fi -# if [ ! $(command -v gh) ] || [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; then -# echo "jq, cf, and gh packages must be installed. Please install via your preferred manager." -# exit 1 -# fi +if [ ! $(command -v gh) ] || [ ! $(command -v jq) ] || [ ! $(command -v cf) ]; then + echo "jq, cf, and gh packages must be installed. Please install via your preferred manager." + exit 1 +fi upcase_name=$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]') -# read -p "Are you on a new branch? We will have to commit this work. (y/n) " -n 1 -r -# echo -# if [[ ! $REPLY =~ ^[Yy]$ ]] -# then -# git checkout -b new-dev-sandbox-$1 -# fi +read -p "Are you on a new branch? We will have to commit this work. (y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + git checkout -b new-dev-sandbox-$1 +fi -# cf target -o cisa-dotgov +cf target -o cisa-dotgov -# read -p "Are you logged in to the cisa-dotgov CF org above? (y/n) " -n 1 -r -# echo -# if [[ ! $REPLY =~ ^[Yy]$ ]] -# then -# cf login -a https://api.fr.cloud.gov --sso -# fi +read -p "Are you logged in to the cisa-dotgov CF org above? (y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + cf login -a https://api.fr.cloud.gov --sso +fi -# gh auth status -# read -p "Are you logged into a Github account with access to cisagov/getgov? (y/n) " -n 1 -r -# echo -# if [[ ! $REPLY =~ ^[Yy]$ ]] -# then -# gh auth login -# fi +gh auth status +read -p "Are you logged into a Github account with access to cisagov/getgov? (y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + gh auth login +fi -# echo "Creating manifest for $1..." -# cp ops/scripts/manifest-sandbox-template.yaml ops/manifests/manifest-$1.yaml -# sed -i '' "s/ENVIRONMENT/$1/" "ops/manifests/manifest-$1.yaml" +echo "Creating manifest for $1..." +cp ops/scripts/manifest-sandbox-template.yaml ops/manifests/manifest-$1.yaml +sed -i '' "s/ENVIRONMENT/$1/" "ops/manifests/manifest-$1.yaml" -# echo "Adding new environment to settings.py..." -# sed -i '' '/getgov-development.app.cloud.gov/ {a\ -# '\"getgov-$1.app.cloud.gov\"', -# }' src/registrar/config/settings.py +echo "Adding new environment to settings.py..." +sed -i '' '/getgov-development.app.cloud.gov/ {a\ + '\"getgov-$1.app.cloud.gov\"', +}' src/registrar/config/settings.py -# echo "Creating new cloud.gov space for $1..." -# cf create-space $1 -# cf target -o "cisa-dotgov" -s $1 -# cf bind-security-group public_networks_egress cisa-dotgov --space $1 -# cf bind-security-group trusted_local_networks_egress cisa-dotgov --space $1 +echo "Creating new cloud.gov space for $1..." +cf create-space $1 +cf target -o "cisa-dotgov" -s $1 +cf bind-security-group public_networks_egress cisa-dotgov --space $1 +cf bind-security-group trusted_local_networks_egress cisa-dotgov --space $1 echo "Creating new cloud.gov DB for $1. This usually takes about 5 minutes..." cf create-service aws-rds micro-psql getgov-$1-database From f0b510f09d68c6d714b4cb26304660211abf4bc6 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 13 Mar 2024 08:48:25 -0700 Subject: [PATCH 081/125] Remove extraneous TODOs except for 1 --- .../management/commands/email_current_metadata_report.py | 4 +--- src/registrar/utility/email.py | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 4300bf227..7e3016842 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -75,8 +75,6 @@ class Command(BaseCommand): # Set context for the subject current_date_str = datetime.now().strftime("%Y-%m-%d") - # TODO: Update secret in getgov-credentials via cloud.gov and my own .env when merging - # Encrypt the metadata encrypted_metadata_in_bytes = self._encrypt_metadata( s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA) @@ -86,7 +84,7 @@ class Command(BaseCommand): send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - # to_address=settings.DEFAULT_FROM_EMAIL, # TODO: Uncomment this when ready to merge + # to_address=settings.DEFAULT_FROM_EMAIL, to_address="rebecca.hsieh@truss.works ", context={"current_date_str": current_date_str}, attachment_file=encrypted_metadata_in_bytes, diff --git a/src/registrar/utility/email.py b/src/registrar/utility/email.py index 91d55f361..5f61181c7 100644 --- a/src/registrar/utility/email.py +++ b/src/registrar/utility/email.py @@ -75,12 +75,9 @@ def send_templated_email( aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, config=settings.BOTO_CONFIG, ) - response = send_email_with_attachment( + send_email_with_attachment( settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, attachment_file, ses_client ) - # TODO: Remove this print statement when ready to merge, - # leaving rn for getting error codes in case - print("Response from send_email_with_attachment_is:", response) except Exception as exc: raise EmailSendingError("Could not send SES email.") from exc From 6dacc18956b114cb08a5d26af2f4906a0f62cd29 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 13 Mar 2024 08:55:54 -0700 Subject: [PATCH 082/125] Update final to email --- .../management/commands/email_current_metadata_report.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 7e3016842..dcaf47b06 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -84,8 +84,7 @@ class Command(BaseCommand): send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - # to_address=settings.DEFAULT_FROM_EMAIL, - to_address="rebecca.hsieh@truss.works ", + to_address=settings.DEFAULT_FROM_EMAIL, context={"current_date_str": current_date_str}, attachment_file=encrypted_metadata_in_bytes, ) From 70bfd4343e9f42eac5761686c6321b99547f8a8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:25:46 -0600 Subject: [PATCH 083/125] Add test, more semaphore(s) --- src/epplibwrapper/client.py | 21 +++-- src/epplibwrapper/tests/test_client.py | 105 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index b346563d2..5ca2b5c26 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -55,15 +55,20 @@ class EPPLibWrapper: ) # We should only ever have one active connection at a time self.connection_lock = BoundedSemaphore(1) + + self.connection_lock.acquire() try: self._initialize_client() except Exception: logger.warning("Unable to configure epplib. Registrar cannot contact registry.") + finally: + self.connection_lock.release() def _initialize_client(self) -> None: """Initialize a client, assuming _login defined. Sets _client to initialized client. Raises errors if initialization fails. This method will be called at app initialization, and also during retries.""" + # establish a client object with a TCP socket transport # note that type: ignore added in several places because linter complains # about _client initially being set to None, and None type doesn't match code @@ -77,11 +82,7 @@ class EPPLibWrapper: ) try: # use the _client object to connect - self._client.connect() # type: ignore - response = self._client.send(self._login) # type: ignore - if response.code >= 2000: # type: ignore - self._client.close() # type: ignore - raise LoginError(response.msg) # type: ignore + self._connect() except TransportError as err: message = "_initialize_client failed to execute due to a connection error." logger.error(f"{message} Error: {err}") @@ -93,6 +94,15 @@ class EPPLibWrapper: logger.error(f"{message} Error: {err}") raise RegistryError(message) from err + def _connect(self) -> None: + """Connects to EPP. Sends a login command. If an invalid response is returned, + the client will be closed and a LoginError raised.""" + self._client.connect() # type: ignore + response = self._client.send(self._login) # type: ignore + if response.code >= 2000: # type: ignore + self._client.close() # type: ignore + raise LoginError(response.msg) # type: ignore + def _disconnect(self) -> None: """Close the connection. Sends a logout command and closes the connection.""" self._send_logout_command() @@ -115,7 +125,6 @@ class EPPLibWrapper: def _send(self, command): """Helper function used by `send`.""" cmd_type = command.__class__.__name__ - try: # check for the condition that the _client was not initialized properly # at app initialization diff --git a/src/epplibwrapper/tests/test_client.py b/src/epplibwrapper/tests/test_client.py index f95b37dcd..17ae6c2cf 100644 --- a/src/epplibwrapper/tests/test_client.py +++ b/src/epplibwrapper/tests/test_client.py @@ -1,4 +1,7 @@ +import datetime +from dateutil.tz import tzlocal # type: ignore from unittest.mock import MagicMock, patch +from pathlib import Path from django.test import TestCase from epplibwrapper.client import EPPLibWrapper from epplibwrapper.errors import RegistryError, LoginError @@ -8,6 +11,10 @@ import logging try: from epplib.exceptions import TransportError from epplib.responses import Result + from epplib.client import Client + from epplib.transport import SocketTransport + from epplib import commands + from epplib.models import common, info except ImportError: pass @@ -255,3 +262,101 @@ class TestClient(TestCase): mock_close.assert_called_once() # send() is called 5 times: send(login), send(command) fail, send(logout), send(login), send(command) self.assertEquals(mock_send.call_count, 5) + + def test_send_command_close_failure_recovers(self): + """Test when the .close on a connection fails and a .send follows suit. + Flow: + Initialization succeeds + Send command fails (with 2400 code) prompting retry + Client closes and re-initializes, and command succeeds""" + + expected_result = { + "cl_tr_id": None, + "code": 1000, + "extensions": [], + "msg": "Command completed successfully", + "msg_q": None, + "res_data": [ + info.InfoDomainResultData( + roid="DF1340360-GOV", + statuses=[ + common.Status( + state="serverTransferProhibited", + description=None, + lang="en", + ), + common.Status(state="inactive", description=None, lang="en"), + ], + cl_id="gov2023-ote", + cr_id="gov2023-ote", + cr_date=datetime.datetime(2023, 8, 15, 23, 56, 36, tzinfo=tzlocal()), + up_id="gov2023-ote", + up_date=datetime.datetime(2023, 8, 17, 2, 3, 19, tzinfo=tzlocal()), + tr_date=None, + name="test3.gov", + registrant="TuaWnx9hnm84GCSU", + admins=[], + nsset=None, + keyset=None, + ex_date=datetime.date(2024, 8, 15), + auth_info=info.DomainAuthInfo(pw="2fooBAR123fooBaz"), + ) + ], + "sv_tr_id": "wRRNVhKhQW2m6wsUHbo/lA==-29a", + } + + def fake_receive(command, cleaned=None): + location = Path(__file__).parent / "utility" / "infoDomain.xml" + xml = (location).read_bytes() + return xml + + def fake_success_send(self, command, cleaned=None): + mock = MagicMock( + code=1000, + msg="Command completed successfully", + res_data=None, + cl_tr_id="xkw1uo#2023-10-17T15:29:09.559376", + sv_tr_id="5CcH4gxISuGkq8eqvr1UyQ==-35a", + extensions=[], + msg_q=None, + ) + return mock + + def fake_failure_send(self, command, cleaned=None): + mock = MagicMock( + code=2400, + msg="Command failed", + res_data=None, + cl_tr_id="xkw1uo#2023-10-17T15:29:09.559376", + sv_tr_id="5CcH4gxISuGkq8eqvr1UyQ==-35a", + extensions=[], + msg_q=None, + ) + return mock + + def do_nothing(command): + pass + + wrapper = None + # Trigger a retry + # Do nothing on connect, as we aren't testing it and want to connect while + # mimicking the rest of the client as closely as possible (which is not entirely possible with MagicMock) + with patch.object(EPPLibWrapper, "_connect", do_nothing): + with patch.object(SocketTransport, "send", fake_failure_send): + wrapper = EPPLibWrapper() + tested_command = commands.InfoDomain(name="test.gov") + try: + wrapper.send(tested_command, cleaned=True) + wrapper._retry(tested_command) + except RegistryError as err: + expected_error = "InfoDomain failed to execute due to a connection error." + self.assertEqual(err.args[0], expected_error) + else: + self.fail("Registry error was not thrown") + + with patch.object(EPPLibWrapper, "_connect", do_nothing): + with patch.object(SocketTransport, "send", fake_success_send), patch.object( + SocketTransport, "receive", fake_receive + ): + result = wrapper.send(tested_command, cleaned=True) + self.assertEqual(expected_result, result.__dict__) From 5cfd6193fd84e5ea39d781666f838e5cbe2cc277 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:28:21 -0600 Subject: [PATCH 084/125] Update client.py --- src/epplibwrapper/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epplibwrapper/client.py b/src/epplibwrapper/client.py index 5ca2b5c26..a130d8bfc 100644 --- a/src/epplibwrapper/client.py +++ b/src/epplibwrapper/client.py @@ -68,7 +68,6 @@ class EPPLibWrapper: """Initialize a client, assuming _login defined. Sets _client to initialized client. Raises errors if initialization fails. This method will be called at app initialization, and also during retries.""" - # establish a client object with a TCP socket transport # note that type: ignore added in several places because linter complains # about _client initially being set to None, and None type doesn't match code @@ -125,6 +124,7 @@ class EPPLibWrapper: def _send(self, command): """Helper function used by `send`.""" cmd_type = command.__class__.__name__ + try: # check for the condition that the _client was not initialized properly # at app initialization From 763a1c51fc52367e11d371a16793c535c9c45198 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:53:30 -0600 Subject: [PATCH 085/125] Merge conflict fixes --- src/registrar/assets/js/get-gov-admin.js | 4 +- .../admin/domain_application_change_form.html | 8 +-- src/registrar/tests/test_admin.py | 58 ------------------- 3 files changed, 6 insertions(+), 64 deletions(-) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 8170e4bd0..4ed00c33f 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -57,7 +57,7 @@ function openInNewTab(el, removeAttribute = false){ createPhantomModalFormButtons(); })(); -/** An IIFE for DomainApplication to hook a modal to a dropdown option. +/** An IIFE for DomainRequest to hook a modal to a dropdown option. * This intentionally does not interact with createPhantomModalFormButtons() */ (function (){ @@ -103,7 +103,7 @@ function openInNewTab(el, removeAttribute = false){ // Because the modal button does not have the class "dja-form-placeholder", // it will not be affected by the createPhantomModalFormButtons() function. - let actionButton = document.querySelector('button[name="_set_application_ineligible"]'); + let actionButton = document.querySelector('button[name="_set_domain_request_ineligible"]'); let valueToCheck = "ineligible" displayModalOnDropdownClick(modalButton, statusDropdown, actionButton, valueToCheck); } diff --git a/src/registrar/templates/django/admin/domain_application_change_form.html b/src/registrar/templates/django/admin/domain_application_change_form.html index f0e4cfe4f..95392da1e 100644 --- a/src/registrar/templates/django/admin/domain_application_change_form.html +++ b/src/registrar/templates/django/admin/domain_application_change_form.html @@ -27,7 +27,7 @@ class="usa-modal" id="toggle-set-ineligible" aria-labelledby="Are you sure you want to select ineligible status?" - aria-describedby="This application will be marked as ineligible." + aria-describedby="This request will be marked as ineligible." >
          @@ -51,7 +51,7 @@ Domain: {{ original.requested_domain.name }} {# Acts as a
          #}
          - New status: {{ original.ApplicationStatus.INELIGIBLE|capfirst }} + New status: {{ original.DomainRequestStatus.INELIGIBLE|capfirst }}

          @@ -61,7 +61,7 @@
        @@ -87,7 +87,7 @@
      • @@ -132,7 +132,7 @@ data-list-one="{{data.unmanaged_domains_sliced_at_start_date}}" data-list-two="{{data.unmanaged_domains_sliced_at_end_date}}" > -

        Chart: Unanaged domains

        +

        Chart: Unmanaged domains

        {{ data.unmanaged_domains_sliced_at_end_date.0 }} unmanaged domains for {{ data.end_date }}

      • diff --git a/src/registrar/templates/admin/app_list.html b/src/registrar/templates/admin/app_list.html index dd7e27f33..49fb59e79 100644 --- a/src/registrar/templates/admin/app_list.html +++ b/src/registrar/templates/admin/app_list.html @@ -64,12 +64,11 @@
        {% endfor %} +
        +

        Analytics

        + Dashboard +
        {% else %}

        {% translate 'You don’t have permission to view or edit anything.' %}

        {% endif %} - -
        -

        Analytics

        - Dashboard -
        diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index f1c7841d1..48b42f47c 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -475,7 +475,7 @@ class AuditedAdminMockData: class MockDb(TestCase): - """Hardcoded mocks make test case assertions sraightforward.""" + """Hardcoded mocks make test case assertions straightforward.""" def setUp(self): super().setUp() @@ -622,19 +622,19 @@ class MockDb(TestCase): ) with less_console_noise(): - self.domain_request_1 = completed_application( + self.domain_request_1 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city1.gov" ) - self.domain_request_2 = completed_application( + self.domain_request_2 = completed_domain_request( status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov" ) - self.domain_request_3 = completed_application( + self.domain_request_3 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city3.gov" ) - self.domain_request_4 = completed_application( + self.domain_request_4 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, name="city4.gov" ) - self.domain_request_5 = completed_application( + self.domain_request_5 = completed_domain_request( status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov" ) self.domain_request_3.submit() diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 475076711..b91f3bd18 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -5,6 +5,8 @@ from io import StringIO from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain from registrar.utility.csv_export import ( + export_data_managed_domains_to_csv, + export_data_unmanaged_domains_to_csv, get_sliced_domains, get_sliced_requests, write_domains_csv, @@ -530,68 +532,10 @@ class ExportDataTest(MockDb, MockEppLib): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.start_date, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - # Call the export functions - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": self.end_date, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_managed_domains_end_date, - get_domain_managers=True, - should_write_header=True, + export_data_managed_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -627,68 +571,10 @@ class ExportDataTest(MockDb, MockEppLib): with less_console_noise(): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.start_date, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) - # Call the export functions - writer.writerow(["UNMANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": self.end_date, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - writer.writerow(["UNMANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - write_domains_csv( - writer, - columns, - sort_fields, - filter_unmanaged_domains_end_date, - get_domain_managers=False, - should_write_header=True, + export_data_unmanaged_domains_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") ) + # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable @@ -696,12 +582,12 @@ class ExportDataTest(MockDb, MockEppLib): self.maxDiff = None # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. expected_content = ( - "UNMANAGED DOMAINS COUNTS AT START DATE\n" + "UNMANAGED DOMAINS AT START DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "0,0,0,0,0,0,0,0,0,0\n" "\n" - "UNMANAGED DOMAINS COUNTS AT END DATE\n" + "UNMANAGED DOMAINS AT END DATE\n" "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," "School district,Election office\n" "1,1,0,0,0,0,0,0,0,0\n" @@ -729,16 +615,17 @@ class ExportDataTest(MockDb, MockEppLib): csv_file = StringIO() writer = csv.writer(csv_file) # Define columns, sort fields, and filter condition + # We'll skip submission date because it's dynamic and therefore + # impossible to set in expected_content columns = [ "Requested domain", "Organization type", - "Submission date", ] sort_fields = [ "requested_domain__name", ] filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, "submission_date__gte": self.start_date, } @@ -750,9 +637,9 @@ class ExportDataTest(MockDb, MockEppLib): # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( - "Requested domain,Organization type,Submission date\n" - "city3.gov,Federal - Executive,2024-03-05\n" - "city4.gov,Federal - Executive,2024-03-05\n" + "Requested domain,Organization type\n" + "city3.gov,Federal - Executive\n" + "city4.gov,Federal - Executive\n" ) # Normalize line endings and remove commas, @@ -785,16 +672,22 @@ class HelperFunctions(MockDb): "domain__permissions__isnull": False, "domain__first_ready__lte": self.end_date, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + # Test with distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition, True) expected_content = [1, 1, 0, 0, 0, 0, 0, 0, 0, 1] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + # Test without distinct + managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + expected_content = [1, 3, 0, 0, 0, 0, 0, 0, 0, 1] + self.assertEqual(managed_domains_sliced_at_end_date, expected_content) + def test_get_sliced_requests(self): """Should get fitered requests counts sliced by org type and election office.""" with less_console_noise(): filter_condition = { - "status": DomainRequest.RequestStatus.SUBMITTED, + "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 060c39804..e8746eafb 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from collections import Counter import csv import logging from datetime import datetime @@ -25,7 +26,8 @@ def write_header(writer, columns): def get_domain_infos(filter_condition, sort_fields): domain_infos = ( - DomainInformation.objects.prefetch_related("domain", "authorizing_official", "domain__permissions") + DomainInformation.objects.select_related("domain", "authorizing_official") + .prefetch_related("domain__permissions") .filter(**filter_condition) .order_by(*sort_fields) .distinct() @@ -190,7 +192,7 @@ def write_domains_csv( def get_requests(filter_condition, sort_fields): - requests = DomainRequest.objects.all().filter(**filter_condition).order_by(*sort_fields).distinct() + requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() return requests @@ -236,10 +238,10 @@ def write_requests_csv( """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. Works with write_header as long as the same writer object is passed.""" - all_requetsts = get_requests(filter_condition, sort_fields) + all_requests = get_requests(filter_condition, sort_fields) # Reduce the memory overhead when performing the write operation - paginator = Paginator(all_requetsts, 1000) + paginator = Paginator(all_requests, 1000) for page_num in paginator.page_range: page = paginator.page(page_num) @@ -443,26 +445,37 @@ def export_data_domain_growth_to_csv(csv_file, start_date, end_date): ) -def get_sliced_domains(filter_condition): - """Get fitered domains counts sliced by org type and election office.""" +def get_sliced_domains(filter_condition, distinct=False): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).count() - state_or_territory = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = domains.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - domains.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = domains.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct domain names based on filter condition + domains_count = DomainInformation.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainInformation.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainInformation.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainInformation.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ domains_count, @@ -478,26 +491,34 @@ def get_sliced_domains(filter_condition): ] -def get_sliced_requests(filter_condition): - """Get fitered requests counts sliced by org type and election office.""" +def get_sliced_requests(filter_condition, distinct=False): + """Get filtered requests counts sliced by org type and election office.""" - requests = DomainRequest.objects.all().filter(**filter_condition).distinct() - requests_count = requests.count() - federal = requests.filter(organization_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(organization_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = requests.filter(organization_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(organization_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(organization_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - requests.filter(organization_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = requests.filter(is_election_board=True).distinct().count() + # Round trip 1: Get distinct requests based on filter condition + requests_count = DomainRequest.objects.filter(**filter_condition).distinct().count() + + # Round trip 2: Get counts for other slices + if distinct: + organization_types_query = ( + DomainRequest.objects.filter(**filter_condition).values_list("organization_type", flat=True).distinct() + ) + else: + organization_types_query = DomainRequest.objects.filter(**filter_condition).values_list( + "organization_type", flat=True + ) + organization_type_counts = Counter(organization_types_query) + + federal = organization_type_counts.get(DomainRequest.OrganizationChoices.FEDERAL, 0) + interstate = organization_type_counts.get(DomainRequest.OrganizationChoices.INTERSTATE, 0) + state_or_territory = organization_type_counts.get(DomainRequest.OrganizationChoices.STATE_OR_TERRITORY, 0) + tribal = organization_type_counts.get(DomainRequest.OrganizationChoices.TRIBAL, 0) + county = organization_type_counts.get(DomainRequest.OrganizationChoices.COUNTY, 0) + city = organization_type_counts.get(DomainRequest.OrganizationChoices.CITY, 0) + special_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SPECIAL_DISTRICT, 0) + school_district = organization_type_counts.get(DomainRequest.OrganizationChoices.SCHOOL_DISTRICT, 0) + + # Round trip 3 + election_board = DomainRequest.objects.filter(is_election_board=True, **filter_condition).distinct().count() return [ requests_count, @@ -531,7 +552,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": start_date_formatted, } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) writer.writerow( @@ -555,7 +576,7 @@ def export_data_managed_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date, True) writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) writer.writerow( @@ -604,7 +625,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": start_date_formatted, } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date, True) writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) writer.writerow( @@ -628,7 +649,7 @@ def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date, True) writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) writer.writerow( diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 04fcaa6f2..eba8423ed 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -2,6 +2,12 @@ from django.http import HttpResponse from django.views import View +from django.shortcuts import render +from django.contrib import admin +from django.db.models import Avg, F +from .. import models +import datetime +from django.utils import timezone from registrar.utility import csv_export @@ -10,6 +16,129 @@ import logging logger = logging.getLogger(__name__) +class AnalyticsView(View): + def get(self, request): + thirty_days_ago = datetime.datetime.today() - datetime.timedelta(days=30) + thirty_days_ago = timezone.make_aware(thirty_days_ago) + + last_30_days_applications = models.DomainRequest.objects.filter(created_at__gt=thirty_days_ago) + last_30_days_approved_applications = models.DomainRequest.objects.filter( + created_at__gt=thirty_days_ago, status=models.DomainRequest.DomainRequestStatus.APPROVED + ) + avg_approval_time = last_30_days_approved_applications.annotate( + approval_time=F("approved_domain__created_at") - F("submission_date") + ).aggregate(Avg("approval_time"))["approval_time__avg"] + # Format the timedelta to display only days + if avg_approval_time is not None: + avg_approval_time_display = f"{avg_approval_time.days} days" + else: + avg_approval_time_display = "No approvals to use" + + # The start and end dates are passed as url params + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + start_date_formatted = csv_export.format_start_date(start_date) + end_date_formatted = csv_export.format_end_date(end_date) + + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date, True) + managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date, True) + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains( + filter_unmanaged_domains_start_date, True + ) + unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date, True) + + filter_ready_domains_start_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": start_date_formatted, + } + filter_ready_domains_end_date = { + "domain__state__in": [models.Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + } + ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + + filter_deleted_domains_start_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": start_date_formatted, + } + filter_deleted_domains_end_date = { + "domain__state__in": [models.Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + } + deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + + filter_requests_start_date = { + "created_at__lte": start_date_formatted, + } + filter_requests_end_date = { + "created_at__lte": end_date_formatted, + } + requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + + filter_submitted_requests_start_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": start_date_formatted, + } + filter_submitted_requests_end_date = { + "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, + "submission_date__lte": end_date_formatted, + } + submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + + context = dict( + # Generate a dictionary of context variables that are common across all admin templates + # (site_header, site_url, ...), + # include it in the larger context dictionary so it's available in the template rendering context. + # This ensures that the admin interface styling and behavior are consistent with other admin pages. + **admin.site.each_context(request), + data=dict( + user_count=models.User.objects.all().count(), + domain_count=models.Domain.objects.all().count(), + ready_domain_count=models.Domain.objects.filter(state=models.Domain.State.READY).count(), + last_30_days_applications=last_30_days_applications.count(), + last_30_days_approved_applications=last_30_days_approved_applications.count(), + average_application_approval_time_last_30_days=avg_approval_time_display, + managed_domains_sliced_at_start_date=managed_domains_sliced_at_start_date, + unmanaged_domains_sliced_at_start_date=unmanaged_domains_sliced_at_start_date, + managed_domains_sliced_at_end_date=managed_domains_sliced_at_end_date, + unmanaged_domains_sliced_at_end_date=unmanaged_domains_sliced_at_end_date, + ready_domains_sliced_at_start_date=ready_domains_sliced_at_start_date, + deleted_domains_sliced_at_start_date=deleted_domains_sliced_at_start_date, + ready_domains_sliced_at_end_date=ready_domains_sliced_at_end_date, + deleted_domains_sliced_at_end_date=deleted_domains_sliced_at_end_date, + requests_sliced_at_start_date=requests_sliced_at_start_date, + submitted_requests_sliced_at_start_date=submitted_requests_sliced_at_start_date, + requests_sliced_at_end_date=requests_sliced_at_end_date, + submitted_requests_sliced_at_end_date=submitted_requests_sliced_at_end_date, + start_date=start_date, + end_date=end_date, + ), + ) + return render(request, "admin/analytics.html", context) + + class ExportDataType(View): def get(self, request, *args, **kwargs): # match the CSV example with all the fields From 9ecd34593c15b439e9b9d09312cd798e6a73969d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 14 Mar 2024 18:31:58 -0400 Subject: [PATCH 100/125] make update charts button stand out more --- src/registrar/templates/admin/analytics.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/admin/analytics.html b/src/registrar/templates/admin/analytics.html index 2c5963e75..e73f22ec5 100644 --- a/src/registrar/templates/admin/analytics.html +++ b/src/registrar/templates/admin/analytics.html @@ -105,7 +105,7 @@
      • -