From d07f7b614b7fb1267cbe8d8da8a5b685f2ddbd6e Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 26 Oct 2023 13:20:45 -0400 Subject: [PATCH 01/38] Slight markup change and some JS to allow for deleting ns 1 and 2 when there are enough records in the formset --- src/registrar/assets/js/get-gov.js | 93 ++++++++++++++----- src/registrar/forms/domain.py | 28 ++++-- .../templates/django/forms/label.html | 12 ++- .../templates/domain_nameservers.html | 20 ++-- 4 files changed, 109 insertions(+), 44 deletions(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index e456575c7..b0062805c 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -256,7 +256,7 @@ function prepareDeleteButtons(formLabel) { let formNumberRegex = RegExp(`form-(\\d){1}-`, 'g'); let formLabelRegex = RegExp(`${formLabel} (\\d+){1}`, 'g'); - // For the eample on Nameservers + // For the example on Nameservers let formExampleRegex = RegExp(`ns(\\d+){1}`, 'g'); forms.forEach((form, index) => { @@ -275,31 +275,35 @@ function prepareDeleteButtons(formLabel) { // h2 and legend for DS form, label for nameservers Array.from(form.querySelectorAll('h2, legend, label, p')).forEach((node) => { - // Ticket: 1192 - // if (isNameserversForm && index <= 1 && !node.innerHTML.includes('*')) { - // // Create a new element - // const newElement = document.createElement('abbr'); - // newElement.textContent = '*'; - // // TODO: finish building abbr + // If the node is a nameserver label, one of the first 2 which was previously 3 and up (not required) + // inject the USWDS required markup and make sure the INPUT is required + if (isNameserversForm && index <= 1 && node.innerHTML.includes('server') && !node.innerHTML.includes('*')) { + // Create a new element + const newElement = document.createElement('abbr'); + newElement.textContent = '*'; + newElement.setAttribute("title", "required"); + newElement.classList.add("usa-hint", "usa-hint--required"); - // // Append the new element to the parent - // node.appendChild(newElement); - // // Find the next sibling that is an input element - // let nextInputElement = node.nextElementSibling; + // Append the new element to the label + node.appendChild(newElement); + // Find the next sibling that is an input element + let nextInputElement = node.nextElementSibling; - // while (nextInputElement) { - // if (nextInputElement.tagName === 'INPUT') { - // // Found the next input element - // console.log(nextInputElement); - // break; - // } - // nextInputElement = nextInputElement.nextElementSibling; - // } - // nextInputElement.required = true; - // } + while (nextInputElement) { + if (nextInputElement.tagName === 'INPUT') { + // Found the next input element + nextInputElement.setAttribute("required", "") + break; + } + nextInputElement = nextInputElement.nextElementSibling; + } + nextInputElement.required = true; + } - // Ticket: 1192 - remove if - if (!(isNameserversForm && index <= 1)) { + let innerSpan = node.querySelector('span') + if (innerSpan) { + innerSpan.textContent = innerSpan.textContent.replace(formLabelRegex, `${formLabel} ${index + 1}`); + } else { node.textContent = node.textContent.replace(formLabelRegex, `${formLabel} ${index + 1}`); node.textContent = node.textContent.replace(formExampleRegex, `ns${index + 1}`); } @@ -309,6 +313,13 @@ function prepareDeleteButtons(formLabel) { if (isNameserversForm && forms.length <= 13) { addButton.classList.remove("display-none") } + + if (isNameserversForm && forms.length < 3) { + // Hide the delete buttons on the remaining nameservers + Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { + deleteButton.classList.add("display-none"); + }); + } }); } @@ -351,6 +362,33 @@ function prepareDeleteButtons(formLabel) { // For the eample on Nameservers let formExampleRegex = RegExp(`ns(\\d){1}`, 'g'); + // Some Nameserver form checks since the delete can mess up the source object we're copying + // in regards to required fileds and hidden delete buttons + if (isNameserversForm) { + + // If the source element we're copying has required on an input, + // reset that input + let formRequiredNeedsCleanUp = newForm.innerHTML.includes('*'); + if (formRequiredNeedsCleanUp) { + newForm.querySelector('label abbr').remove(); + // Get all input elements within the container + const inputElements = newForm.querySelectorAll("input"); + // Loop through each input element and remove the 'required' attribute + inputElements.forEach((input) => { + if (input.hasAttribute("required")) { + input.removeAttribute("required"); + } + }); + } + + // If the source element we're copying has an invisible delete button, + // show that button + let deleteButtonNeedsCleanUp = newForm.querySelector('.delete-record').classList.contains("display-none"); + if (deleteButtonNeedsCleanUp) { + newForm.querySelector('.delete-record').classList.remove("display-none"); + } + } + formNum++; newForm.innerHTML = newForm.innerHTML.replace(formNumberRegex, `form-${formNum-1}-`); newForm.innerHTML = newForm.innerHTML.replace(formLabelRegex, `${formLabel} ${formNum}`); @@ -404,6 +442,15 @@ function prepareDeleteButtons(formLabel) { if (isNameserversForm && formNum == 13) { addButton.classList.add("display-none") } + + if (isNameserversForm && forms.length >= 2) { + // Show the delete buttons on the nameservers + forms.forEach((form, index) => { + Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { + deleteButton.classList.remove("display-none"); + }); + }); + } } })(); diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index c7ebd0807..82fd7d30e 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,8 +1,11 @@ """Forms for domain management.""" +from collections.abc import Mapping +from typing import Any from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator from django.forms import formset_factory +from django.forms.utils import ErrorList from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.utility.errors import ( @@ -23,20 +26,31 @@ class DomainAddUserForm(forms.Form): email = forms.EmailField(label="Email") -class IPAddressField(forms.CharField): - def validate(self, value): - super().validate(value) # Run the default CharField validation - - class DomainNameserverForm(forms.Form): """Form for changing nameservers.""" domain = forms.CharField(widget=forms.HiddenInput, required=False) - server = forms.CharField(label="Name server", strip=True) + server = forms.CharField( + label="Name server", + strip=True + + ) - ip = forms.CharField(label="IP Address (IPv4 or IPv6)", strip=True, required=False) + ip = forms.CharField( + label="IP Address (IPv4 or IPv6)", + strip=True, + required=False, + ) + + def __init__(self, *args, **kwargs): + super(DomainNameserverForm, self).__init__(*args, **kwargs) + # add custom error messages + self.fields['server'].error_messages.update({ + 'required': 'A minimum of 2 Name Servers are required.', + }) + def clean(self): # clean is called from clean_forms, which is called from is_valid # after clean_fields. it is used to determine form level errors. diff --git a/src/registrar/templates/django/forms/label.html b/src/registrar/templates/django/forms/label.html index 6881922f2..da90a372a 100644 --- a/src/registrar/templates/django/forms/label.html +++ b/src/registrar/templates/django/forms/label.html @@ -2,8 +2,12 @@ class="{% if label_classes %} {{ label_classes }}{% endif %}{% if label_tag == 'legend' %} {{ legend_classes }}{% endif %}" {% if not field.use_fieldset %}for="{{ widget.attrs.id }}"{% endif %} > -{{ field.label }} -{% if widget.attrs.required %} - * -{% endif %} + {% if span_for_text %} + {{ field.label }} + {% else %} + {{ field.label }} + {% endif %} + {% if widget.attrs.required %} + * + {% endif %} diff --git a/src/registrar/templates/domain_nameservers.html b/src/registrar/templates/domain_nameservers.html index d00126698..76e78d7ac 100644 --- a/src/registrar/templates/domain_nameservers.html +++ b/src/registrar/templates/domain_nameservers.html @@ -35,11 +35,14 @@ {{ form.domain }} {% with sublabel_text="Example: ns"|concat:forloop.counter|concat:".example.com" %} {% if forloop.counter <= 2 %} - {% with attr_required=True add_group_class="usa-form-group--unstyled-error" %} + {# span_for_text will wrap the copy in s , which we'll use in the JS for this component #} + {% with attr_required=True add_group_class="usa-form-group--unstyled-error" span_for_text=True %} {% input_with_errors form.server %} {% endwith %} {% else %} - {% input_with_errors form.server %} + {% with span_for_text=True %} + {% input_with_errors form.server %} + {% endwith %} {% endif %} {% endwith %} @@ -49,14 +52,11 @@ {% endwith %}
- {% comment %} TODO: remove this if for 1192 {% endcomment %} - {% if forloop.counter > 2 %} - - {% endif %} +
From 9dd2e733cc4fb962654b228490dec0e9372e1403 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 26 Oct 2023 14:39:27 -0400 Subject: [PATCH 02/38] instead of show/hide, enable/disable --- src/registrar/assets/js/get-gov.js | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index b0062805c..489cdcad2 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -311,13 +311,14 @@ function prepareDeleteButtons(formLabel) { // Remove the add more button if we have less than 13 forms if (isNameserversForm && forms.length <= 13) { - addButton.classList.remove("display-none") + addButton.classList.remove("display-none"); } if (isNameserversForm && forms.length < 3) { // Hide the delete buttons on the remaining nameservers Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { - deleteButton.classList.add("display-none"); + // deleteButton.classList.add("display-none"); + deleteButton.setAttribute("disabled", "true"); }); } @@ -381,11 +382,12 @@ function prepareDeleteButtons(formLabel) { }); } - // If the source element we're copying has an invisible delete button, - // show that button - let deleteButtonNeedsCleanUp = newForm.querySelector('.delete-record').classList.contains("display-none"); - if (deleteButtonNeedsCleanUp) { - newForm.querySelector('.delete-record').classList.remove("display-none"); + // If the source element we're copying has an disabled delete button, + // enable that button + let deleteButton= newForm.querySelector('.delete-record'); + if (deleteButton.hasAttribute("disabled")) { + // newForm.querySelector('.delete-record').classList.remove("display-none"); + deleteButton.removeAttribute("disabled"); } } @@ -438,16 +440,18 @@ function prepareDeleteButtons(formLabel) { // Attach click event listener on the delete buttons of the new form prepareDeleteButtons(formLabel); - // Hide the add more button if we have 13 forms + // Disable the add more button if we have 13 forms if (isNameserversForm && formNum == 13) { - addButton.classList.add("display-none") + // addButton.classList.add("display-none"); + addButton.setAttribute("disabled", "true"); } if (isNameserversForm && forms.length >= 2) { - // Show the delete buttons on the nameservers + // Enable the delete buttons on the nameservers forms.forEach((form, index) => { Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { - deleteButton.classList.remove("display-none"); + // deleteButton.classList.remove("display-none"); + deleteButton.removeAttribute("disabled"); }); }); } From ee2b001ab55d1ecd3e9f1006de7fa1bf411be59a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 26 Oct 2023 16:17:25 -0400 Subject: [PATCH 03/38] infrastructure for buttons, actions and a csv download --- src/registrar/admin.py | 30 +++++++++++++++++++ .../django/admin/domain_change_list.html | 25 ++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/registrar/templates/django/admin/domain_change_list.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 904ce66a4..feae93ca7 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,5 +1,7 @@ +import csv import logging from django import forms +from django.http import HttpResponse from django_fsm import get_available_FIELD_transitions from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin @@ -747,7 +749,35 @@ class DomainAdmin(ListHeaderAdmin): search_fields = ["name"] search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" + change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state"] + # actions = ['export_data'] + + # Define a custom view for data export + def export_static_data(self, request): + # Your data export logic here + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + writer = csv.writer(response) + # Write your data to the CSV here + writer.writerow(['Name', 'State', ...]) # Include the appropriate headers + # Loop through and write your data rows + for data_row in Domain.objects.all(): + writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + return response + + def get_urls(self): + from django.urls import path + + urlpatterns = super().get_urls() + + info = self.model._meta.app_label, self.model._meta.model_name + + my_url = [ + path('export-static-data/', self.export_static_data, name='%s_%s_export_static_data' % info), + ] + + return my_url + urlpatterns def response_change(self, request, obj): # Create dictionary of action functions diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html new file mode 100644 index 000000000..f21216052 --- /dev/null +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -0,0 +1,25 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools %} + +
    + {% if has_add_permission %} +
  • + + Add Domain + +
    • +
  • + {% comment %} + {% endcomment %} + Export Data +
    • +
  • + +
    • +
  • + +
    • + {% endif %} +
+{% endblock %} \ No newline at end of file From 5fe28275653f98a6733e73efad772143a871fb00 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 26 Oct 2023 17:11:17 -0400 Subject: [PATCH 04/38] rough out the first quesry and export method based on an example report --- src/registrar/admin.py | 72 +++++++++++++++++-- .../django/admin/domain_change_list.html | 8 +-- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index feae93ca7..364aaf04a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -11,6 +11,7 @@ from django.http.response import HttpResponseRedirect from django.urls import reverse from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.domain import Domain +from registrar.models.domain_information import DomainInformation from registrar.models.utility.admin_sort_fields import AdminSortFields from . import models from auditlog.models import LogEntry # type: ignore @@ -751,11 +752,70 @@ class DomainAdmin(ListHeaderAdmin): change_form_template = "django/admin/domain_change_form.html" change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state"] - # actions = ['export_data'] - # Define a custom view for data export - def export_static_data(self, request): - # Your data export logic here + def export_data_type(self, request): + # match the CSV example with all the fields + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + writer = csv.writer(response) + # Write your data to the CSV here + writer.writerow( + [ + 'Domain name', + 'Domain type', + 'Federal agency', + 'Organization name', + 'City', + 'State', + 'AO', + 'AO email', + 'Submitter', + 'Submitter title', + 'Submitter email', + 'Submitter phone', + 'Security Contact Email', + 'Status', + # 'Expiration Date' + ] + ) # Include the appropriate headers + # Loop through and write your data rows + for domain in Domain.objects.all(): + domain_information, _ = DomainInformation.objects.get_or_create(domain=domain) + writer.writerow( + [ + domain.name, + domain_information.federal_type, + domain_information.federal_agency, + domain_information.organization_name, + domain_information.city, + domain_information.state_territory, + domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, + domain_information.authorizing_official.email, + domain_information.submitter.first_name + " " + domain_information.submitter.last_name, + domain_information.submitter.title, + domain_information.submitter.email, + domain_information.submitter.phone, + domain.security_contact.email if domain.security_contact else " ", + domain.state, + # domain.expiration_date, + ] + ) # Include the appropriate fields + return response + + def export_data_full(self, request): + # Smaller export based on 1 + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + writer = csv.writer(response) + # Write your data to the CSV here + writer.writerow(['Name', 'State', ...]) # Include the appropriate headers + # Loop through and write your data rows + for data_row in Domain.objects.all(): + writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + return response + + def export_data_federal(self, request): + # Federal only response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="data_export.csv"' writer = csv.writer(response) @@ -774,7 +834,9 @@ class DomainAdmin(ListHeaderAdmin): info = self.model._meta.app_label, self.model._meta.model_name my_url = [ - path('export-static-data/', self.export_static_data, name='%s_%s_export_static_data' % info), + path('export_data_type/', self.export_data_type, name='%s_%s_export_data_type' % info), + path('export_data_full/', self.export_data_full, name='%s_%s_export_data_full' % info), + path('export_data_federal/', self.export_data_federal, name='%s_%s_export_data_federal' % info), ] return my_url + urlpatterns diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index f21216052..e3a142477 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -10,15 +10,13 @@
  • - {% comment %} - {% endcomment %} - Export Data + Export domains (type)
  • - + Export domains (full)
  • - + Export domains (federal)
    • {% endif %} From e4b3706d9c44934f345b3ad74d3b9421b46f4d2f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 27 Oct 2023 05:55:52 -0400 Subject: [PATCH 05/38] refactored some for separation of concerns, and fixing permissions in template --- src/registrar/admin.py | 63 ++--------------- .../django/admin/domain_change_list.html | 2 +- src/registrar/utility/csv_export.py | 68 +++++++++++++++++++ 3 files changed, 74 insertions(+), 59 deletions(-) create mode 100644 src/registrar/utility/csv_export.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 364aaf04a..c4ff83214 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,4 +1,3 @@ -import csv import logging from django import forms from django.http import HttpResponse @@ -11,8 +10,8 @@ from django.http.response import HttpResponseRedirect from django.urls import reverse from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.domain import Domain -from registrar.models.domain_information import DomainInformation from registrar.models.utility.admin_sort_fields import AdminSortFields +from registrar.utility import csv_export from . import models from auditlog.models import LogEntry # type: ignore from auditlog.admin import LogEntryAdmin # type: ignore @@ -757,73 +756,21 @@ class DomainAdmin(ListHeaderAdmin): # match the CSV example with all the fields response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="data_export.csv"' - writer = csv.writer(response) - # Write your data to the CSV here - writer.writerow( - [ - 'Domain name', - 'Domain type', - 'Federal agency', - 'Organization name', - 'City', - 'State', - 'AO', - 'AO email', - 'Submitter', - 'Submitter title', - 'Submitter email', - 'Submitter phone', - 'Security Contact Email', - 'Status', - # 'Expiration Date' - ] - ) # Include the appropriate headers - # Loop through and write your data rows - for domain in Domain.objects.all(): - domain_information, _ = DomainInformation.objects.get_or_create(domain=domain) - writer.writerow( - [ - domain.name, - domain_information.federal_type, - domain_information.federal_agency, - domain_information.organization_name, - domain_information.city, - domain_information.state_territory, - domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, - domain_information.authorizing_official.email, - domain_information.submitter.first_name + " " + domain_information.submitter.last_name, - domain_information.submitter.title, - domain_information.submitter.email, - domain_information.submitter.phone, - domain.security_contact.email if domain.security_contact else " ", - domain.state, - # domain.expiration_date, - ] - ) # Include the appropriate fields + csv_export.export_data_type_to_csv(response) return response - + def export_data_full(self, request): # Smaller export based on 1 response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="data_export.csv"' - writer = csv.writer(response) - # Write your data to the CSV here - writer.writerow(['Name', 'State', ...]) # Include the appropriate headers - # Loop through and write your data rows - for data_row in Domain.objects.all(): - writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + csv_export.export_data_full_to_csv(response) return response def export_data_federal(self, request): # Federal only response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="data_export.csv"' - writer = csv.writer(response) - # Write your data to the CSV here - writer.writerow(['Name', 'State', ...]) # Include the appropriate headers - # Loop through and write your data rows - for data_row in Domain.objects.all(): - writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + csv_export.export_data_federal_to_csv(response) return response def get_urls(self): diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index e3a142477..9eb624891 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -9,6 +9,7 @@ Add Domain + {% endif %}
  • Export domains (type)
    • @@ -18,6 +19,5 @@
  • Export domains (federal)
    • - {% endif %} {% endblock %} \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py new file mode 100644 index 000000000..c7fb9a0f1 --- /dev/null +++ b/src/registrar/utility/csv_export.py @@ -0,0 +1,68 @@ +import csv +from registrar.models.domain import Domain +from registrar.models.domain_information import DomainInformation +from registrar.models.public_contact import PublicContact + +def export_data_type_to_csv(csv_file): + writer = csv.writer(csv_file) + # Write your data to the CSV here + writer.writerow( + [ + 'Domain name', + 'Domain type', + 'Federal agency', + 'Organization name', + 'City', + 'State', + 'AO', + 'AO email', + 'Submitter', + 'Submitter title', + 'Submitter email', + 'Submitter phone', + 'Security Contact Email', + 'Status', + # 'Expiration Date' + ] + ) # Include the appropriate headers + # Loop through and write your data rows + for domain in Domain.objects.all().order_by('name'): + domain_information, _ = DomainInformation.objects.get_or_create(domain=domain) + security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + + writer.writerow( + [ + domain.name, + domain_information.federal_type, + domain_information.federal_agency, + domain_information.organization_name, + domain_information.city, + domain_information.state_territory, + domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, + domain_information.authorizing_official.email, + domain_information.submitter.first_name + " " + domain_information.submitter.last_name, + domain_information.submitter.title, + domain_information.submitter.email, + domain_information.submitter.phone, + security_contacts[0].email if security_contacts.exists() else " ", + # domain.contacts.all().filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)[0].email if len(domain.contacts.all().filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)) else " ", + domain.state, + # domain.expiration_date, + ] + ) # Include the appropriate fields + +def export_data_full_to_csv(csv_file): + writer = csv.writer(csv_file) + # Write your data to the CSV here + writer.writerow(['Name', 'State', ...]) # Include the appropriate headers + # Loop through and write your data rows + for data_row in Domain.objects.all(): + writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + +def export_data_federal_to_csv(csv_file): + writer = csv.writer(csv_file) + # Write your data to the CSV here + writer.writerow(['Name', 'State', ...]) # Include the appropriate headers + # Loop through and write your data rows + for data_row in Domain.objects.all(): + writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields \ No newline at end of file From 0b947ce75427cbe8c3f980a4e3d94a7334f48f92 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 27 Oct 2023 06:58:11 -0400 Subject: [PATCH 06/38] modularization of reporting functions --- src/registrar/admin.py | 1 + src/registrar/utility/csv_export.py | 129 +++++++++++++++++----------- 2 files changed, 78 insertions(+), 52 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c4ff83214..e61ddc38e 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -752,6 +752,7 @@ class DomainAdmin(ListHeaderAdmin): change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state"] + # TODO file names and function names specific to report type def export_data_type(self, request): # match the CSV example with all the fields response = HttpResponse(content_type='text/csv') diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index c7fb9a0f1..ac28cd61d 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -3,66 +3,91 @@ from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact -def export_data_type_to_csv(csv_file): - writer = csv.writer(csv_file) - # Write your data to the CSV here - writer.writerow( - [ - 'Domain name', - 'Domain type', - 'Federal agency', - 'Organization name', - 'City', - 'State', - 'AO', - 'AO email', - 'Submitter', - 'Submitter title', - 'Submitter email', - 'Submitter phone', - 'Security Contact Email', - 'Status', - # 'Expiration Date' - ] - ) # Include the appropriate headers - # Loop through and write your data rows +# TODO pass sort order and filter as arguments rather than domains +def export_domains_to_writer(writer, domains, columns): + # write columns headers to writer + writer.writerow(columns) + for domain in Domain.objects.all().order_by('name'): domain_information, _ = DomainInformation.objects.get_or_create(domain=domain) security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + # create a dictionary of fields to include + FIELDS = { + 'Domain name': domain.name, + 'Domain type': domain_information.federal_type, + 'Federal agency': domain_information.federal_agency, + 'Organization name': domain_information.organization_name, + 'City': domain_information.city, + 'State': domain_information.state_territory, + 'AO': domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, + 'AO email': domain_information.authorizing_official.email, + 'Submitter': domain_information.submitter.first_name + " " + domain_information.submitter.last_name, + 'Submitter title': domain_information.submitter.title, + 'Submitter email': domain_information.submitter.email, + 'Submitter phone': domain_information.submitter.phone, + 'Security Contact Email': security_contacts[0].email if security_contacts.exists() else " ", + 'Status': domain.state, + } writer.writerow( - [ - domain.name, - domain_information.federal_type, - domain_information.federal_agency, - domain_information.organization_name, - domain_information.city, - domain_information.state_territory, - domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, - domain_information.authorizing_official.email, - domain_information.submitter.first_name + " " + domain_information.submitter.last_name, - domain_information.submitter.title, - domain_information.submitter.email, - domain_information.submitter.phone, - security_contacts[0].email if security_contacts.exists() else " ", - # domain.contacts.all().filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)[0].email if len(domain.contacts.all().filter(contact_type=PublicContact.ContactTypeChoices.SECURITY)) else " ", - domain.state, - # domain.expiration_date, - ] - ) # Include the appropriate fields + [FIELDS.get(column,'') for column in columns] + ) + +def export_data_type_to_csv(csv_file): + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + 'Domain name', + 'Domain type', + 'Federal agency', + 'Organization name', + 'City', + 'State', + 'AO', + 'AO email', + 'Submitter', + 'Submitter title', + 'Submitter email', + 'Submitter phone', + 'Security Contact Email', + 'Status', + # 'Expiration Date' + ] + # define domains to be exported + domains = Domain.objects.all().order_by('name') + export_domains_to_writer(writer, domains, columns) def export_data_full_to_csv(csv_file): writer = csv.writer(csv_file) - # Write your data to the CSV here - writer.writerow(['Name', 'State', ...]) # Include the appropriate headers - # Loop through and write your data rows - for data_row in Domain.objects.all(): - writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields + # define columns to include in export + columns = [ + 'Domain name', + 'Domain type', + 'Federal agency', + 'Organization name', + 'City', + 'State', + 'Security Contact Email', + ] + # define domains to be exported + # TODO order by fields in domain information + domains = Domain.objects.all().order_by('name') + export_domains_to_writer(writer, domains, columns) def export_data_federal_to_csv(csv_file): writer = csv.writer(csv_file) - # Write your data to the CSV here - writer.writerow(['Name', 'State', ...]) # Include the appropriate headers - # Loop through and write your data rows - for data_row in Domain.objects.all(): - writer.writerow([data_row.name, data_row.state, ...]) # Include the appropriate fields \ No newline at end of file + # define columns to include in export + columns = [ + 'Domain name', + 'Domain type', + 'Federal agency', + 'Organization name', + 'City', + 'State', + 'Security Contact Email', + ] + # define domains to be exported + # TODO order by fields in domain information + # TODO filter by domain type + domains = Domain.objects.all().order_by('name') + export_domains_to_writer(writer, domains, columns) \ No newline at end of file From 47251d9edb4017224164fc4e8ec79f54f4110c43 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 27 Oct 2023 08:06:46 -0400 Subject: [PATCH 07/38] parameterize filters and sorts --- src/registrar/admin.py | 7 ++-- src/registrar/utility/csv_export.py | 59 +++++++++++++---------------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index e61ddc38e..3b641d9df 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -752,25 +752,24 @@ class DomainAdmin(ListHeaderAdmin): change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state"] - # TODO file names and function names specific to report type def export_data_type(self, request): # match the CSV example with all the fields response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + response['Content-Disposition'] = 'attachment; filename="domains-by-type.csv"' csv_export.export_data_type_to_csv(response) return response def export_data_full(self, request): # Smaller export based on 1 response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + response['Content-Disposition'] = 'attachment; filename="domains-current-full.csv"' csv_export.export_data_full_to_csv(response) return response def export_data_federal(self, request): # Federal only response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="data_export.csv"' + response['Content-Disposition'] = 'attachment; filename="domains-current-federal.csv"' csv_export.export_data_federal_to_csv(response) return response diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index ac28cd61d..ff60f8ff3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,33 +1,31 @@ import csv -from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact -# TODO pass sort order and filter as arguments rather than domains -def export_domains_to_writer(writer, domains, columns): +def export_domains_to_writer(writer, columns, sort_fields, filter_condition): # write columns headers to writer writer.writerow(columns) - for domain in Domain.objects.all().order_by('name'): - domain_information, _ = DomainInformation.objects.get_or_create(domain=domain) - security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + domainInfos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields) + for domainInfo in domainInfos: + security_contacts = domainInfo.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) - # create a dictionary of fields to include + # create a dictionary of fields which can be included in output FIELDS = { - 'Domain name': domain.name, - 'Domain type': domain_information.federal_type, - 'Federal agency': domain_information.federal_agency, - 'Organization name': domain_information.organization_name, - 'City': domain_information.city, - 'State': domain_information.state_territory, - 'AO': domain_information.authorizing_official.first_name + " " + domain_information.authorizing_official.last_name, - 'AO email': domain_information.authorizing_official.email, - 'Submitter': domain_information.submitter.first_name + " " + domain_information.submitter.last_name, - 'Submitter title': domain_information.submitter.title, - 'Submitter email': domain_information.submitter.email, - 'Submitter phone': domain_information.submitter.phone, + 'Domain name': domainInfo.domain.name, + 'Domain type': domainInfo.organization_type, + 'Federal agency': domainInfo.federal_agency, + 'Organization name': domainInfo.organization_name, + 'City': domainInfo.city, + 'State': domainInfo.state_territory, + 'AO': domainInfo.authorizing_official.first_name + " " + domainInfo.authorizing_official.last_name, + 'AO email': domainInfo.authorizing_official.email, + 'Submitter': domainInfo.submitter.first_name + " " + domainInfo.submitter.last_name, + 'Submitter title': domainInfo.submitter.title, + 'Submitter email': domainInfo.submitter.email, + 'Submitter phone': domainInfo.submitter.phone, 'Security Contact Email': security_contacts[0].email if security_contacts.exists() else " ", - 'Status': domain.state, + 'Status': domainInfo.domain.state, } writer.writerow( [FIELDS.get(column,'') for column in columns] @@ -53,9 +51,9 @@ def export_data_type_to_csv(csv_file): 'Status', # 'Expiration Date' ] - # define domains to be exported - domains = Domain.objects.all().order_by('name') - export_domains_to_writer(writer, domains, columns) + sort_fields = ['domain__name'] + filter_condition = {} + export_domains_to_writer(writer, columns, sort_fields, filter_condition) def export_data_full_to_csv(csv_file): writer = csv.writer(csv_file) @@ -69,10 +67,9 @@ def export_data_full_to_csv(csv_file): 'State', 'Security Contact Email', ] - # define domains to be exported - # TODO order by fields in domain information - domains = Domain.objects.all().order_by('name') - export_domains_to_writer(writer, domains, columns) + sort_fields = ['domain__name', 'federal_agency', 'organization_type'] + filter_condition = {} + export_domains_to_writer(writer, columns, sort_fields, filter_condition) def export_data_federal_to_csv(csv_file): writer = csv.writer(csv_file) @@ -86,8 +83,6 @@ def export_data_federal_to_csv(csv_file): 'State', 'Security Contact Email', ] - # define domains to be exported - # TODO order by fields in domain information - # TODO filter by domain type - domains = Domain.objects.all().order_by('name') - export_domains_to_writer(writer, domains, columns) \ No newline at end of file + sort_fields = ['domain__name', 'federal_agency', 'organization_type'] + filter_condition = {'organization_type__icontains': 'federal'} + export_domains_to_writer(writer, columns, sort_fields, filter_condition) \ No newline at end of file From b72919a892438bd3fc5f4ec1963d190159782568 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 27 Oct 2023 08:11:13 -0400 Subject: [PATCH 08/38] formatting for readability --- src/registrar/admin.py | 42 ++++++---- src/registrar/utility/csv_export.py | 116 +++++++++++++++------------- 2 files changed, 93 insertions(+), 65 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3b641d9df..53ca897be 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -751,39 +751,55 @@ class DomainAdmin(ListHeaderAdmin): change_form_template = "django/admin/domain_change_form.html" change_list_template = "django/admin/domain_change_list.html" readonly_fields = ["state"] - + def export_data_type(self, request): # match the CSV example with all the fields - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="domains-by-type.csv"' + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' csv_export.export_data_type_to_csv(response) return response def export_data_full(self, request): # Smaller export based on 1 - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="domains-current-full.csv"' + response = HttpResponse(content_type="text/csv") + response[ + "Content-Disposition" + ] = 'attachment; filename="domains-current-full.csv"' csv_export.export_data_full_to_csv(response) return response - + def export_data_federal(self, request): # Federal only - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="domains-current-federal.csv"' + response = HttpResponse(content_type="text/csv") + response[ + "Content-Disposition" + ] = 'attachment; filename="domains-current-federal.csv"' csv_export.export_data_federal_to_csv(response) return response - + def get_urls(self): from django.urls import path - + urlpatterns = super().get_urls() info = self.model._meta.app_label, self.model._meta.model_name my_url = [ - path('export_data_type/', self.export_data_type, name='%s_%s_export_data_type' % info), - path('export_data_full/', self.export_data_full, name='%s_%s_export_data_full' % info), - path('export_data_federal/', self.export_data_federal, name='%s_%s_export_data_federal' % info), + path( + "export_data_type/", + self.export_data_type, + name="%s_%s_export_data_type" % info, + ), + path( + "export_data_full/", + self.export_data_full, + name="%s_%s_export_data_full" % info, + ), + path( + "export_data_federal/", + self.export_data_federal, + name="%s_%s_export_data_federal" % info, + ), ] return my_url + urlpatterns diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index ff60f8ff3..fbb2325ff 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -2,87 +2,99 @@ import csv from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact + def export_domains_to_writer(writer, columns, sort_fields, filter_condition): # write columns headers to writer writer.writerow(columns) - domainInfos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields) + domainInfos = DomainInformation.objects.filter(**filter_condition).order_by( + *sort_fields + ) for domainInfo in domainInfos: - security_contacts = domainInfo.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + security_contacts = domainInfo.domain.contacts.filter( + contact_type=PublicContact.ContactTypeChoices.SECURITY + ) # create a dictionary of fields which can be included in output FIELDS = { - 'Domain name': domainInfo.domain.name, - 'Domain type': domainInfo.organization_type, - 'Federal agency': domainInfo.federal_agency, - 'Organization name': domainInfo.organization_name, - 'City': domainInfo.city, - 'State': domainInfo.state_territory, - 'AO': domainInfo.authorizing_official.first_name + " " + domainInfo.authorizing_official.last_name, - 'AO email': domainInfo.authorizing_official.email, - 'Submitter': domainInfo.submitter.first_name + " " + domainInfo.submitter.last_name, - 'Submitter title': domainInfo.submitter.title, - 'Submitter email': domainInfo.submitter.email, - 'Submitter phone': domainInfo.submitter.phone, - 'Security Contact Email': security_contacts[0].email if security_contacts.exists() else " ", - 'Status': domainInfo.domain.state, + "Domain name": domainInfo.domain.name, + "Domain type": domainInfo.organization_type, + "Federal agency": domainInfo.federal_agency, + "Organization name": domainInfo.organization_name, + "City": domainInfo.city, + "State": domainInfo.state_territory, + "AO": domainInfo.authorizing_official.first_name + + " " + + domainInfo.authorizing_official.last_name, + "AO email": domainInfo.authorizing_official.email, + "Submitter": domainInfo.submitter.first_name + + " " + + domainInfo.submitter.last_name, + "Submitter title": domainInfo.submitter.title, + "Submitter email": domainInfo.submitter.email, + "Submitter phone": domainInfo.submitter.phone, + "Security Contact Email": security_contacts[0].email + if security_contacts.exists() + else " ", + "Status": domainInfo.domain.state, } - writer.writerow( - [FIELDS.get(column,'') for column in columns] - ) + writer.writerow([FIELDS.get(column, "") for column in columns]) + def export_data_type_to_csv(csv_file): writer = csv.writer(csv_file) # define columns to include in export columns = [ - 'Domain name', - 'Domain type', - 'Federal agency', - 'Organization name', - 'City', - 'State', - 'AO', - 'AO email', - 'Submitter', - 'Submitter title', - 'Submitter email', - 'Submitter phone', - 'Security Contact Email', - 'Status', + "Domain name", + "Domain type", + "Federal agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Submitter", + "Submitter title", + "Submitter email", + "Submitter phone", + "Security Contact Email", + "Status", # 'Expiration Date' ] - sort_fields = ['domain__name'] + sort_fields = ["domain__name"] filter_condition = {} export_domains_to_writer(writer, columns, sort_fields, filter_condition) + def export_data_full_to_csv(csv_file): writer = csv.writer(csv_file) # define columns to include in export columns = [ - 'Domain name', - 'Domain type', - 'Federal agency', - 'Organization name', - 'City', - 'State', - 'Security Contact Email', + "Domain name", + "Domain type", + "Federal agency", + "Organization name", + "City", + "State", + "Security Contact Email", ] - sort_fields = ['domain__name', 'federal_agency', 'organization_type'] + sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = {} export_domains_to_writer(writer, columns, sort_fields, filter_condition) + def export_data_federal_to_csv(csv_file): writer = csv.writer(csv_file) # define columns to include in export columns = [ - 'Domain name', - 'Domain type', - 'Federal agency', - 'Organization name', - 'City', - 'State', - 'Security Contact Email', + "Domain name", + "Domain type", + "Federal agency", + "Organization name", + "City", + "State", + "Security Contact Email", ] - sort_fields = ['domain__name', 'federal_agency', 'organization_type'] - filter_condition = {'organization_type__icontains': 'federal'} - export_domains_to_writer(writer, columns, sort_fields, filter_condition) \ No newline at end of file + sort_fields = ["domain__name", "federal_agency", "organization_type"] + filter_condition = {"organization_type__icontains": "federal"} + export_domains_to_writer(writer, columns, sort_fields, filter_condition) From 09c0a5f3920f4b67f1e742d5639f16492503e292 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 11:35:06 -0400 Subject: [PATCH 09/38] match names to old reports, change buttons copy --- src/registrar/admin.py | 4 ++-- .../templates/django/admin/domain_change_list.html | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 53ca897be..1286f2025 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -764,7 +764,7 @@ class DomainAdmin(ListHeaderAdmin): response = HttpResponse(content_type="text/csv") response[ "Content-Disposition" - ] = 'attachment; filename="domains-current-full.csv"' + ] = 'attachment; filename="current-full.csv"' csv_export.export_data_full_to_csv(response) return response @@ -773,7 +773,7 @@ class DomainAdmin(ListHeaderAdmin): response = HttpResponse(content_type="text/csv") response[ "Content-Disposition" - ] = 'attachment; filename="domains-current-federal.csv"' + ] = 'attachment; filename="current-federal.csv"' csv_export.export_data_federal_to_csv(response) return response diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index 9eb624891..45626c55b 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -11,13 +11,13 @@ {% endif %}
  • - Export domains (type) + Export all domain metadata
  • - Export domains (full) + Export current-full.csv
  • - Export domains (federal) + Export current-federal.csv
    • {% endblock %} \ No newline at end of file From 26947d4bd2529f8d59b8d831d8c215d093aeafeb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 12:50:30 -0400 Subject: [PATCH 10/38] filter by READY, edge case handling --- .../django/admin/domain_change_list.html | 14 ++++---- src/registrar/utility/csv_export.py | 33 +++++++++++++------ 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index 45626c55b..b99f71e2f 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -3,13 +3,6 @@ {% block object-tools %} {% endblock %} \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index fbb2325ff..146cc9ce3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,4 +1,5 @@ import csv +from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact @@ -25,16 +26,28 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): "State": domainInfo.state_territory, "AO": domainInfo.authorizing_official.first_name + " " - + domainInfo.authorizing_official.last_name, - "AO email": domainInfo.authorizing_official.email, + + domainInfo.authorizing_official.last_name + if domainInfo.authorizing_official + else " ", + "AO email": domainInfo.authorizing_official.email + if domainInfo.authorizing_official + else " ", "Submitter": domainInfo.submitter.first_name + " " - + domainInfo.submitter.last_name, - "Submitter title": domainInfo.submitter.title, - "Submitter email": domainInfo.submitter.email, - "Submitter phone": domainInfo.submitter.phone, + + domainInfo.submitter.last_name + if domainInfo.submitter + else " ", + "Submitter title": domainInfo.submitter.title + if domainInfo.submitter + else " ", + "Submitter email": domainInfo.submitter.email + if domainInfo.submitter + else " ", + "Submitter phone": domainInfo.submitter.phone + if domainInfo.submitter + else " ", "Security Contact Email": security_contacts[0].email - if security_contacts.exists() + if security_contacts else " ", "Status": domainInfo.domain.state, } @@ -62,7 +75,7 @@ def export_data_type_to_csv(csv_file): # 'Expiration Date' ] sort_fields = ["domain__name"] - filter_condition = {} + filter_condition = {"domain__state": Domain.State.READY} export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -79,7 +92,7 @@ def export_data_full_to_csv(csv_file): "Security Contact Email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] - filter_condition = {} + filter_condition = {"domain__state": Domain.State.READY} export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -96,5 +109,5 @@ def export_data_federal_to_csv(csv_file): "Security Contact Email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] - filter_condition = {"organization_type__icontains": "federal"} + filter_condition = {"organization_type__icontains": "federal", "domain__state": Domain.State.READY} export_domains_to_writer(writer, columns, sort_fields, filter_condition) From 98a9604ef372f09875aab2e46285bc2dded366cf Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 14:50:07 -0400 Subject: [PATCH 11/38] Unit tests for reports --- src/registrar/tests/test_reports.py | 162 ++++++++++++++++++++++++++++ src/registrar/tests/test_views.py | 1 + 2 files changed, 163 insertions(+) create mode 100644 src/registrar/tests/test_reports.py diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py new file mode 100644 index 000000000..d2dee1d5a --- /dev/null +++ b/src/registrar/tests/test_reports.py @@ -0,0 +1,162 @@ +from django.test import TestCase +from django.core.files import File +from io import StringIO +import csv +from registrar.models.domain_information import DomainInformation +from registrar.models.domain import Domain +from registrar.models.user import User +from django.contrib.auth import get_user_model +from registrar.utility.csv_export import ( + export_domains_to_writer, + export_data_type_to_csv, + export_data_full_to_csv, + export_data_federal_to_csv, +) + +class ExportDataTest(TestCase): + def setUp(self): + + username = "test_user" + first_name = "First" + last_name = "Last" + email = "info@example.com" + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, email=email + ) + + self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) + self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.READY) + self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.READY) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + + self.domain_information_1, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_1, + organization_type="federal", + federal_agency="World War I Centennial Commission", + ) + self.domain_information_2, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_2, + organization_type="interstate", + ) + self.domain_information_3, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_3, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_4, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_4, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + + def tearDown(self): + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + User.objects.all().delete() + super().tearDown() + + + def test_export_domains_to_writer(self): + """Test that export_domains_to_writer returns the + existing domain, test that sort by domain name works, + test that filter works""" + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + "Federal agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Submitter", + "Submitter title", + "Submitter email", + "Submitter phone", + "Security Contact Email", + "Status", + ] + sort_fields = ["domain__name"] + filter_condition = {"domain__state": Domain.State.READY} + + # Call the export function + export_domains_to_writer(writer, columns, sort_fields, filter_condition) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = """\ +Domain name,Domain type,Federal agency,Organization name,City,State,AO,AO email,Submitter,Submitter title,Submitter email,Submitter phone,Security Contact Email,Status +adomain2.gov,interstate,,,,, , , , , , , ,ready +cdomain1.gov,federal,World War I Centennial Commission,,,, , , , , , , ,ready +ddomain3.gov,federal,Armed Forces Retirement Home,,,, , , , , , , ,ready +""" + # print(csv_content) + # self.maxDiff = None + + # Normalize line endings and remove leading/trailing whitespace + csv_content = csv_content.replace('\r\n', '\n').strip() + expected_content = expected_content.strip() + + self.assertEqual(csv_content, expected_content) + + def test_export_domains_to_writer_2(self): + """An additional test for filters and multi-column sort""" + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + "Federal agency", + "Organization name", + "City", + "State", + "Security Contact Email", + ] + sort_fields = ["domain__name", "federal_agency", "organization_type"] + filter_condition = {"organization_type__icontains": "federal", "domain__state": Domain.State.READY} + + # Call the export function + export_domains_to_writer(writer, columns, sort_fields, filter_condition) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains, + # federal only + # sorted alphabetially by domain name + expected_content = """\ +Domain name,Domain type,Federal agency,Organization name,City,State,Security Contact Email +cdomain1.gov,federal,World War I Centennial Commission,,,, +ddomain3.gov,federal,Armed Forces Retirement Home,,,, +""" + # print(csv_content) + # self.maxDiff = None + + # Normalize line endings and remove leading/trailing whitespace + csv_content = csv_content.replace('\r\n', '\n').strip() + expected_content = expected_content.strip() + + self.assertEqual(csv_content, expected_content) + diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index bff64eb33..0db177f05 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1139,6 +1139,7 @@ class TestWithDomainPermissions(TestWithUser): if hasattr(self.domain, "contacts"): self.domain.contacts.all().delete() DomainApplication.objects.all().delete() + DomainInformation.objects.all().delete() PublicContact.objects.all().delete() Domain.objects.all().delete() UserDomainRole.objects.all().delete() From 4803aaf971fca2b243a8742023c2930ba7d0d04e Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 15:03:34 -0400 Subject: [PATCH 12/38] lint --- src/registrar/admin.py | 8 +-- src/registrar/tests/test_reports.py | 97 ++++++++++++++++------------- src/registrar/utility/csv_export.py | 5 +- 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 1286f2025..8abf7a53b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -762,18 +762,14 @@ class DomainAdmin(ListHeaderAdmin): def export_data_full(self, request): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") - response[ - "Content-Disposition" - ] = 'attachment; filename="current-full.csv"' + response["Content-Disposition"] = 'attachment; filename="current-full.csv"' csv_export.export_data_full_to_csv(response) return response def export_data_federal(self, request): # Federal only response = HttpResponse(content_type="text/csv") - response[ - "Content-Disposition" - ] = 'attachment; filename="current-federal.csv"' + response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' csv_export.export_data_federal_to_csv(response) return response diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index d2dee1d5a..4a2b2ccfd 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -1,21 +1,15 @@ from django.test import TestCase -from django.core.files import File from io import StringIO import csv from registrar.models.domain_information import DomainInformation from registrar.models.domain import Domain from registrar.models.user import User from django.contrib.auth import get_user_model -from registrar.utility.csv_export import ( - export_domains_to_writer, - export_data_type_to_csv, - export_data_full_to_csv, - export_data_federal_to_csv, -) +from registrar.utility.csv_export import export_domains_to_writer + class ExportDataTest(TestCase): def setUp(self): - username = "test_user" first_name = "First" last_name = "Last" @@ -24,12 +18,22 @@ class ExportDataTest(TestCase): username=username, first_name=first_name, last_name=last_name, email=email ) - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) - self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.READY) - self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.READY) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY + ) + self.domain_2, _ = Domain.objects.get_or_create( + name="adomain2.gov", state=Domain.State.READY + ) + self.domain_3, _ = Domain.objects.get_or_create( + name="ddomain3.gov", state=Domain.State.READY + ) + self.domain_4, _ = Domain.objects.get_or_create( + name="bdomain4.gov", state=Domain.State.UNKNOWN + ) + self.domain_4, _ = Domain.objects.get_or_create( + name="bdomain4.gov", state=Domain.State.UNKNOWN + ) + self.domain_information_1, _ = DomainInformation.objects.get_or_create( creator=self.user, domain=self.domain_1, @@ -53,14 +57,13 @@ class ExportDataTest(TestCase): organization_type="federal", federal_agency="Armed Forces Retirement Home", ) - + def tearDown(self): Domain.objects.all().delete() DomainInformation.objects.all().delete() User.objects.all().delete() super().tearDown() - def test_export_domains_to_writer(self): """Test that export_domains_to_writer returns the existing domain, test that sort by domain name works, @@ -68,7 +71,7 @@ class ExportDataTest(TestCase): # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - + # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -94,33 +97,36 @@ class ExportDataTest(TestCase): # Reset the CSV file's position to the beginning csv_file.seek(0) - + # Read the content into a variable csv_content = csv_file.read() - + # We expect READY domains, # sorted alphabetially by domain name - expected_content = """\ -Domain name,Domain type,Federal agency,Organization name,City,State,AO,AO email,Submitter,Submitter title,Submitter email,Submitter phone,Security Contact Email,Status -adomain2.gov,interstate,,,,, , , , , , , ,ready -cdomain1.gov,federal,World War I Centennial Commission,,,, , , , , , , ,ready -ddomain3.gov,federal,Armed Forces Retirement Home,,,, , , , , , , ,ready -""" + expected_content = ( + "Domain name,Domain type,Federal agency,Organization name,City,State,AO," + "AO email, Submitter,Submitter title,Submitter email,Submitter phone," + "Security Contact Email,Status\n" + "adomain2.gov,interstate,,,,, , , , , , , ,ready\n" + "cdomain1.gov,federal,World War I Centennial Commission,,," + ", , , , , , , ,ready\n" + "ddomain3.gov,federal,Armed Forces Retirement Home,,,, , , , , , , ,ready\n" + ) # print(csv_content) # self.maxDiff = None - + # Normalize line endings and remove leading/trailing whitespace - csv_content = csv_content.replace('\r\n', '\n').strip() + csv_content = csv_content.replace("\r\n", "\n").strip() expected_content = expected_content.strip() - + self.assertEqual(csv_content, expected_content) - + def test_export_domains_to_writer_2(self): """An additional test for filters and multi-column sort""" # Create a CSV file in memory csv_file = StringIO() writer = csv.writer(csv_file) - + # Define columns, sort fields, and filter condition columns = [ "Domain name", @@ -132,31 +138,34 @@ ddomain3.gov,federal,Armed Forces Retirement Home,,,, , , , , , , ,ready "Security Contact Email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] - filter_condition = {"organization_type__icontains": "federal", "domain__state": Domain.State.READY} + filter_condition = { + "organization_type__icontains": "federal", + "domain__state": Domain.State.READY, + } # Call the export function export_domains_to_writer(writer, columns, sort_fields, filter_condition) # Reset the CSV file's position to the beginning csv_file.seek(0) - + # Read the content into a variable csv_content = csv_file.read() - + # We expect READY domains, # federal only # sorted alphabetially by domain name - expected_content = """\ -Domain name,Domain type,Federal agency,Organization name,City,State,Security Contact Email -cdomain1.gov,federal,World War I Centennial Commission,,,, -ddomain3.gov,federal,Armed Forces Retirement Home,,,, -""" + expected_content = ( + "Domain name,Domain type,Federal agency,Organization name,City," + "State,Security Contact Email\n" + "cdomain1.gov,federal,World War I Centennial Commission,,,,\n" + "ddomain3.gov,federal,Armed Forces Retirement Home,,,,\n" + ) # print(csv_content) # self.maxDiff = None - - # Normalize line endings and remove leading/trailing whitespace - csv_content = csv_content.replace('\r\n', '\n').strip() - expected_content = expected_content.strip() - - self.assertEqual(csv_content, expected_content) + # Normalize line endings and remove leading/trailing whitespace + csv_content = csv_content.replace("\r\n", "\n").strip() + expected_content = expected_content.strip() + + self.assertEqual(csv_content, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 146cc9ce3..1d8725611 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -109,5 +109,8 @@ def export_data_federal_to_csv(csv_file): "Security Contact Email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] - filter_condition = {"organization_type__icontains": "federal", "domain__state": Domain.State.READY} + filter_condition = { + "organization_type__icontains": "federal", + "domain__state": Domain.State.READY, + } export_domains_to_writer(writer, columns, sort_fields, filter_condition) From 677743cbf3d4357e3bccedd72aa899f3b817fceb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 15:39:32 -0400 Subject: [PATCH 13/38] tweak expected_content with an extra space to match actual_content --- src/registrar/tests/test_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 4a2b2ccfd..f6837aa83 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -105,7 +105,7 @@ class ExportDataTest(TestCase): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Federal agency,Organization name,City,State,AO," - "AO email, Submitter,Submitter title,Submitter email,Submitter phone," + "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" "adomain2.gov,interstate,,,,, , , , , , , ,ready\n" "cdomain1.gov,federal,World War I Centennial Commission,,," @@ -158,7 +158,7 @@ class ExportDataTest(TestCase): expected_content = ( "Domain name,Domain type,Federal agency,Organization name,City," "State,Security Contact Email\n" - "cdomain1.gov,federal,World War I Centennial Commission,,,,\n" + "cdomain1.gov,federal,World War I Centennial Commission,,,, \n" "ddomain3.gov,federal,Armed Forces Retirement Home,,,,\n" ) # print(csv_content) From 91420429c2f3d4f9c5131f0eb5aa169ddecea2f2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 17:05:43 -0400 Subject: [PATCH 14/38] remove submitter fields: --- src/registrar/utility/csv_export.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1d8725611..2888814b1 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -32,20 +32,6 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): "AO email": domainInfo.authorizing_official.email if domainInfo.authorizing_official else " ", - "Submitter": domainInfo.submitter.first_name - + " " - + domainInfo.submitter.last_name - if domainInfo.submitter - else " ", - "Submitter title": domainInfo.submitter.title - if domainInfo.submitter - else " ", - "Submitter email": domainInfo.submitter.email - if domainInfo.submitter - else " ", - "Submitter phone": domainInfo.submitter.phone - if domainInfo.submitter - else " ", "Security Contact Email": security_contacts[0].email if security_contacts else " ", @@ -66,10 +52,6 @@ def export_data_type_to_csv(csv_file): "State", "AO", "AO email", - "Submitter", - "Submitter title", - "Submitter email", - "Submitter phone", "Security Contact Email", "Status", # 'Expiration Date' From d78c3e063ffb3f33668879bf5bc5f0979831adbb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 17:07:20 -0400 Subject: [PATCH 15/38] lint --- src/registrar/forms/domain.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 82fd7d30e..5d792e7ac 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -31,26 +31,24 @@ class DomainNameserverForm(forms.Form): domain = forms.CharField(widget=forms.HiddenInput, required=False) - server = forms.CharField( - label="Name server", - strip=True - - ) + server = forms.CharField(label="Name server", strip=True) ip = forms.CharField( label="IP Address (IPv4 or IPv6)", strip=True, required=False, ) - + def __init__(self, *args, **kwargs): super(DomainNameserverForm, self).__init__(*args, **kwargs) # add custom error messages - self.fields['server'].error_messages.update({ - 'required': 'A minimum of 2 Name Servers are required.', - }) - + self.fields["server"].error_messages.update( + { + "required": "A minimum of 2 Name Servers are required.", + } + ) + def clean(self): # clean is called from clean_forms, which is called from is_valid # after clean_fields. it is used to determine form level errors. From af5cfca94e173351c4da7fac7bf54c3154749d15 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 17:26:13 -0400 Subject: [PATCH 16/38] Fix tests --- src/registrar/tests/test_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 1560f2203..9e0a66545 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1406,7 +1406,7 @@ class TestDomainNameservers(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the required field. form requires a minimum of 2 name servers - self.assertContains(result, "This field is required.", count=2, status_code=200) + self.assertContains(result, "A minimum of 2 Name Servers are required.", count=2, status_code=200) def test_domain_nameservers_form_submit_subdomain_missing_ip(self): """Can change domain's nameservers. @@ -1573,7 +1573,7 @@ class TestDomainNameservers(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears four times, twice at the top of the page, # once around each required field. - self.assertContains(result, "This field is required", count=4, status_code=200) + self.assertContains(result, "A minimum of 2 Name Servers are required.", count=4, status_code=200) class TestDomainAuthorizingOfficial(TestDomainOverview): From 87bdcb8263c349628e197b5ba50677923e2c620f Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 18:34:06 -0400 Subject: [PATCH 17/38] Normalize line endings and remove commas, spaces and leading/trailing whitespace --- src/registrar/tests/test_reports.py | 25 +++++++++++-------------- src/registrar/utility/csv_export.py | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index f6837aa83..608ff09cd 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -107,17 +107,16 @@ class ExportDataTest(TestCase): "Domain name,Domain type,Federal agency,Organization name,City,State,AO," "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" - "adomain2.gov,interstate,,,,, , , , , , , ,ready\n" - "cdomain1.gov,federal,World War I Centennial Commission,,," - ", , , , , , , ,ready\n" - "ddomain3.gov,federal,Armed Forces Retirement Home,,,, , , , , , , ,ready\n" + "adomain2.gov,interstate,ready\n" + "cdomain1.gov,federal,World War I Centennial Commission,ready\n" + "ddomain3.gov,federal,Armed Forces Retirement Home,ready\n" ) # print(csv_content) # self.maxDiff = None - # Normalize line endings and remove leading/trailing whitespace - csv_content = csv_content.replace("\r\n", "\n").strip() - expected_content = expected_content.strip() + # Normalize line endings and remove commas, spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() #noqa + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() #noqa self.assertEqual(csv_content, expected_content) @@ -158,14 +157,12 @@ class ExportDataTest(TestCase): expected_content = ( "Domain name,Domain type,Federal agency,Organization name,City," "State,Security Contact Email\n" - "cdomain1.gov,federal,World War I Centennial Commission,,,, \n" - "ddomain3.gov,federal,Armed Forces Retirement Home,,,,\n" + "cdomain1.gov,federal,World War I Centennial Commission\n" + "ddomain3.gov,federal,Armed Forces Retirement Home\n" ) - # print(csv_content) - # self.maxDiff = None - # Normalize line endings and remove leading/trailing whitespace - csv_content = csv_content.replace("\r\n", "\n").strip() - expected_content = expected_content.strip() + # Normalize line endings and remove commas, spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() #noqa + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() #noqa self.assertEqual(csv_content, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 2888814b1..846404995 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -57,7 +57,11 @@ def export_data_type_to_csv(csv_file): # 'Expiration Date' ] sort_fields = ["domain__name"] - filter_condition = {"domain__state": Domain.State.READY} + filter_condition = { + "domain__state": Domain.State.READY, + "domain__state": Domain.State.DNS_NEEDED, + "domain__state": Domain.State.ON_HOLD, + } export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -74,7 +78,11 @@ def export_data_full_to_csv(csv_file): "Security Contact Email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] - filter_condition = {"domain__state": Domain.State.READY} + filter_condition = { + "domain__state": Domain.State.READY, + "domain__state": Domain.State.DNS_NEEDED, + "domain__state": Domain.State.ON_HOLD, + } export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -94,5 +102,7 @@ def export_data_federal_to_csv(csv_file): filter_condition = { "organization_type__icontains": "federal", "domain__state": Domain.State.READY, + "domain__state": Domain.State.DNS_NEEDED, + "domain__state": Domain.State.ON_HOLD, } export_domains_to_writer(writer, columns, sort_fields, filter_condition) From 272890012b7b5a1fe34a0a3419afe135d0fb30db Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 18:40:07 -0400 Subject: [PATCH 18/38] lint --- src/registrar/tests/test_reports.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 608ff09cd..a0674b3c3 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -115,8 +115,16 @@ class ExportDataTest(TestCase): # self.maxDiff = None # Normalize line endings and remove commas, spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() #noqa - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() #noqa + csv_content = ( + csv_content.replace(",,", "") + .replace(",", "") + .replace(" ", "") + .replace("\r\n", "\n") + .strip() + ) + expected_content = ( + expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + ) self.assertEqual(csv_content, expected_content) @@ -162,7 +170,15 @@ class ExportDataTest(TestCase): ) # Normalize line endings and remove commas, spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() #noqa - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() #noqa + csv_content = ( + csv_content.replace(",,", "") + .replace(",", "") + .replace(" ", "") + .replace("\r\n", "\n") + .strip() + ) + expected_content = ( + expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + ) self.assertEqual(csv_content, expected_content) From 82772c633b745a054a6485d0d18f5f85b9f74edc Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 27 Oct 2023 18:57:13 -0400 Subject: [PATCH 19/38] lint --- src/registrar/tests/test_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index a0674b3c3..2f9155423 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -176,7 +176,7 @@ class ExportDataTest(TestCase): .replace(" ", "") .replace("\r\n", "\n") .strip() - ) + ) expected_content = ( expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() ) From e6e6cfa24f8f8ebf83cfe7f157ee7e37b60c5168 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sun, 29 Oct 2023 06:04:35 -0400 Subject: [PATCH 20/38] update to filter_conditions; formatting for lines too long --- src/registrar/tests/test_reports.py | 6 ++++-- src/registrar/utility/csv_export.py | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 2f9155423..7474194a3 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -114,7 +114,8 @@ class ExportDataTest(TestCase): # print(csv_content) # self.maxDiff = None - # Normalize line endings and remove commas, spaces and leading/trailing whitespace + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace csv_content = ( csv_content.replace(",,", "") .replace(",", "") @@ -169,7 +170,8 @@ class ExportDataTest(TestCase): "ddomain3.gov,federal,Armed Forces Retirement Home\n" ) - # Normalize line endings and remove commas, spaces and leading/trailing whitespace + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace csv_content = ( csv_content.replace(",,", "") .replace(",", "") diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 846404995..281fc6078 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -58,9 +58,11 @@ def export_data_type_to_csv(csv_file): ] sort_fields = ["domain__name"] filter_condition = { - "domain__state": Domain.State.READY, - "domain__state": Domain.State.DNS_NEEDED, - "domain__state": Domain.State.ON_HOLD, + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], } export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -79,9 +81,11 @@ def export_data_full_to_csv(csv_file): ] sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { - "domain__state": Domain.State.READY, - "domain__state": Domain.State.DNS_NEEDED, - "domain__state": Domain.State.ON_HOLD, + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], } export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -101,8 +105,10 @@ def export_data_federal_to_csv(csv_file): sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { "organization_type__icontains": "federal", - "domain__state": Domain.State.READY, - "domain__state": Domain.State.DNS_NEEDED, - "domain__state": Domain.State.ON_HOLD, + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], } export_domains_to_writer(writer, columns, sort_fields, filter_condition) From 534e281eeb42a96424de0be7eeb752bbe51947df Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 12:13:48 -0400 Subject: [PATCH 21/38] linter --- src/registrar/tests/test_views.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 9e0a66545..06666ebd4 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1406,7 +1406,12 @@ class TestDomainNameservers(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the required field. form requires a minimum of 2 name servers - self.assertContains(result, "A minimum of 2 Name Servers are required.", count=2, status_code=200) + self.assertContains( + result, + "A minimum of 2 Name Servers are required.", + count=2, + status_code=200, + ) def test_domain_nameservers_form_submit_subdomain_missing_ip(self): """Can change domain's nameservers. @@ -1573,7 +1578,12 @@ class TestDomainNameservers(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears four times, twice at the top of the page, # once around each required field. - self.assertContains(result, "A minimum of 2 Name Servers are required.", count=4, status_code=200) + self.assertContains( + result, + "A minimum of 2 Name Servers are required.", + count=4, + status_code=200, + ) class TestDomainAuthorizingOfficial(TestDomainOverview): From 3b3757c13eea9f50deabae04457ca5dc6e667286 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 12:19:40 -0400 Subject: [PATCH 22/38] lint --- src/registrar/forms/domain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 5d792e7ac..01e5adeb9 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -1,11 +1,8 @@ """Forms for domain management.""" -from collections.abc import Mapping -from typing import Any from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator from django.forms import formset_factory -from django.forms.utils import ErrorList from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.utility.errors import ( From 8b2ea986b1ef752db62d3e70fbda395bb23a6cde Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 18:52:23 -0400 Subject: [PATCH 23/38] Change generic connection error message, test for 13 nameservers on pageload --- src/registrar/assets/js/get-gov.js | 11 ++++++----- src/registrar/forms/domain.py | 2 +- src/registrar/templates/domain_nameservers.html | 2 +- src/registrar/tests/test_views.py | 12 ++++++++---- src/registrar/utility/errors.py | 8 +++++--- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index 3aa3a736b..27090d747 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -315,7 +315,6 @@ function prepareDeleteButtons(formLabel) { if (isNameserversForm && forms.length < 3) { // Hide the delete buttons on the remaining nameservers Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { - // deleteButton.classList.add("display-none"); deleteButton.setAttribute("disabled", "true"); }); } @@ -345,6 +344,11 @@ function prepareDeleteButtons(formLabel) { formLabel = "DS Data record"; } + // On load: Disable the add more button if we have 13 forms + if (isNameserversForm && document.querySelectorAll(".repeatable-form").length == 13) { + addButton.setAttribute("disabled", "true"); + } + // Attach click event listener on the delete buttons of the existing forms prepareDeleteButtons(formLabel); @@ -361,7 +365,7 @@ function prepareDeleteButtons(formLabel) { let formExampleRegex = RegExp(`ns(\\d){1}`, 'g'); // Some Nameserver form checks since the delete can mess up the source object we're copying - // in regards to required fileds and hidden delete buttons + // in regards to required fields and hidden delete buttons if (isNameserversForm) { // If the source element we're copying has required on an input, @@ -383,7 +387,6 @@ function prepareDeleteButtons(formLabel) { // enable that button let deleteButton= newForm.querySelector('.delete-record'); if (deleteButton.hasAttribute("disabled")) { - // newForm.querySelector('.delete-record').classList.remove("display-none"); deleteButton.removeAttribute("disabled"); } } @@ -439,7 +442,6 @@ function prepareDeleteButtons(formLabel) { // Disable the add more button if we have 13 forms if (isNameserversForm && formNum == 13) { - // addButton.classList.add("display-none"); addButton.setAttribute("disabled", "true"); } @@ -447,7 +449,6 @@ function prepareDeleteButtons(formLabel) { // Enable the delete buttons on the nameservers forms.forEach((form, index) => { Array.from(form.querySelectorAll('.delete-record')).forEach((deleteButton) => { - // deleteButton.classList.remove("display-none"); deleteButton.removeAttribute("disabled"); }); }); diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index e486a66a2..a6173d376 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -42,7 +42,7 @@ class DomainNameserverForm(forms.Form): # add custom error messages self.fields["server"].error_messages.update( { - "required": "A minimum of 2 Name Servers are required.", + "required": "A minimum of 2 name servers are required.", } ) diff --git a/src/registrar/templates/domain_nameservers.html b/src/registrar/templates/domain_nameservers.html index 76e78d7ac..55519ef67 100644 --- a/src/registrar/templates/domain_nameservers.html +++ b/src/registrar/templates/domain_nameservers.html @@ -81,7 +81,7 @@ type="submit" class="usa-button usa-button--outline" name="btn-cancel-click" - aria-label="Reset the data in the Name Server form to the registry state (undo changes)" + aria-label="Reset the data in the name server form to the registry state (undo changes)" >Cancel diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 94d407a40..02f19cff0 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1466,7 +1466,7 @@ class TestDomainNameservers(TestDomainOverview): # the required field. form requires a minimum of 2 name servers self.assertContains( result, - "A minimum of 2 Name Servers are required.", + "A minimum of 2 name servers are required.", count=2, status_code=200, ) @@ -1639,7 +1639,7 @@ class TestDomainNameservers(TestDomainOverview): # once around each required field. self.assertContains( result, - "A minimum of 2 Name Servers are required.", + "A minimum of 2 name servers are required.", count=4, status_code=200, ) @@ -1810,7 +1810,11 @@ class TestDomainSecurityEmail(TestDomainOverview): ( "RegistryError", form_data_registry_error, - "Update failed. Cannot contact the registry.", + """ +We’re experiencing a system connection error. Please wait a few minutes +and try again. If you continue to receive this error after a few tries, +contact help@get.gov + """, ), ("ContactError", form_data_contact_error, "Value entered was wrong."), ( @@ -1845,7 +1849,7 @@ class TestDomainSecurityEmail(TestDomainOverview): self.assertEqual(len(messages), 1) message = messages[0] self.assertEqual(message.tags, message_tag) - self.assertEqual(message.message, expected_message) + self.assertEqual(message.message.strip(), expected_message.strip()) def test_domain_overview_blocked_for_ineligible_user(self): """We could easily duplicate this test for all domain management diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index c1d3c5849..5d62953ac 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -39,9 +39,11 @@ class GenericError(Exception): """ _error_mapping = { - GenericErrorCodes.CANNOT_CONTACT_REGISTRY: ( - "Update failed. Cannot contact the registry." - ), + GenericErrorCodes.CANNOT_CONTACT_REGISTRY: """ +We’re experiencing a system connection error. Please wait a few minutes +and try again. If you continue to receive this error after a few tries, +contact help@get.gov + """, GenericErrorCodes.GENERIC_ERROR: ("Value entered was wrong."), } From 4c677458698e7d15457c3d08032dc1150b495d15 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 19:07:54 -0400 Subject: [PATCH 24/38] enable addbutton when conditions are right (instead of show) --- src/registrar/assets/js/get-gov.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index 27090d747..fa9c8d8ea 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -309,7 +309,7 @@ function prepareDeleteButtons(formLabel) { // Display the add more button if we have less than 13 forms if (isNameserversForm && forms.length <= 13) { - addButton.classList.remove("display-none"); + addButton.classList.removeAttribute("disabled"); } if (isNameserversForm && forms.length < 3) { From d624c220883c4b23f564145a7e5c42b39e84f39f Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 19:28:26 -0400 Subject: [PATCH 25/38] enhance tests with more domain statuses --- src/registrar/tests/test_reports.py | 24 ++++++++++++++++-------- src/registrar/utility/csv_export.py | 3 ++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 7474194a3..3f6b36041 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -22,10 +22,10 @@ class ExportDataTest(TestCase): name="cdomain1.gov", state=Domain.State.READY ) self.domain_2, _ = Domain.objects.get_or_create( - name="adomain2.gov", state=Domain.State.READY + name="adomain2.gov", state=Domain.State.DNS_NEEDED ) self.domain_3, _ = Domain.objects.get_or_create( - name="ddomain3.gov", state=Domain.State.READY + name="ddomain3.gov", state=Domain.State.ON_HOLD ) self.domain_4, _ = Domain.objects.get_or_create( name="bdomain4.gov", state=Domain.State.UNKNOWN @@ -90,7 +90,13 @@ class ExportDataTest(TestCase): "Status", ] sort_fields = ["domain__name"] - filter_condition = {"domain__state": Domain.State.READY} + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } # Call the export function export_domains_to_writer(writer, columns, sort_fields, filter_condition) @@ -107,12 +113,10 @@ class ExportDataTest(TestCase): "Domain name,Domain type,Federal agency,Organization name,City,State,AO," "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" - "adomain2.gov,interstate,ready\n" + "adomain2.gov,interstate,dnsneeded\n" "cdomain1.gov,federal,World War I Centennial Commission,ready\n" - "ddomain3.gov,federal,Armed Forces Retirement Home,ready\n" + "ddomain3.gov,federal,Armed Forces Retirement Home,onhold\n" ) - # print(csv_content) - # self.maxDiff = None # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -148,7 +152,11 @@ class ExportDataTest(TestCase): sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { "organization_type__icontains": "federal", - "domain__state": Domain.State.READY, + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], } # Call the export function diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 281fc6078..e6bfb7334 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -36,6 +36,7 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): if security_contacts else " ", "Status": domainInfo.domain.state, + "Expiration Date": domainInfo.domain.expiration_date, } writer.writerow([FIELDS.get(column, "") for column in columns]) @@ -54,7 +55,7 @@ def export_data_type_to_csv(csv_file): "AO email", "Security Contact Email", "Status", - # 'Expiration Date' + "Expiration Date", ] sort_fields = ["domain__name"] filter_condition = { From c82b870d165b8a2a43c3e7597e34aed530aa8498 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Mon, 30 Oct 2023 19:31:53 -0400 Subject: [PATCH 26/38] fix JS bug where classList.removeAttribute --- src/registrar/assets/js/get-gov.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index fa9c8d8ea..b659b117e 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -309,7 +309,8 @@ function prepareDeleteButtons(formLabel) { // Display the add more button if we have less than 13 forms if (isNameserversForm && forms.length <= 13) { - addButton.classList.removeAttribute("disabled"); + console.log('remove disabled'); + addButton.removeAttribute("disabled"); } if (isNameserversForm && forms.length < 3) { From 940df2eb6850b31c0a40d25987f49df6f46f2e41 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 10:14:52 -0400 Subject: [PATCH 27/38] Fix capitalization on IP address --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index a6173d376..d48a14c6b 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -31,7 +31,7 @@ class DomainNameserverForm(forms.Form): server = forms.CharField(label="Name server", strip=True) ip = forms.CharField( - label="IP Address (IPv4 or IPv6)", + label="IP address (IPv4 or IPv6)", strip=True, required=False, ) From 8e768112f3d55e072f5b4b9cc4d9fb9b5a861589 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 10:52:00 -0400 Subject: [PATCH 28/38] Tweak domain type display by using get_organization_type_display and pulling federal_type if applicable --- src/registrar/tests/test_reports.py | 10 +++++----- src/registrar/utility/csv_export.py | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 3f6b36041..81380c929 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -113,9 +113,9 @@ class ExportDataTest(TestCase): "Domain name,Domain type,Federal agency,Organization name,City,State,AO," "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" - "adomain2.gov,interstate,dnsneeded\n" - "cdomain1.gov,federal,World War I Centennial Commission,ready\n" - "ddomain3.gov,federal,Armed Forces Retirement Home,onhold\n" + "adomain2.gov,Interstate,dnsneeded\n" + "cdomain1.gov,Federal,World War I Centennial Commission,ready\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,onhold\n" ) # Normalize line endings and remove commas, @@ -174,8 +174,8 @@ class ExportDataTest(TestCase): expected_content = ( "Domain name,Domain type,Federal agency,Organization name,City," "State,Security Contact Email\n" - "cdomain1.gov,federal,World War I Centennial Commission\n" - "ddomain3.gov,federal,Armed Forces Retirement Home\n" + "cdomain1.gov,Federal,World War I Centennial Commission\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home\n" ) # Normalize line endings and remove commas, diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index e6bfb7334..339a42111 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -19,7 +19,10 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): # create a dictionary of fields which can be included in output FIELDS = { "Domain name": domainInfo.domain.name, - "Domain type": domainInfo.organization_type, + "Domain type": domainInfo.get_organization_type_display() + + " - " + domainInfo.federal_type + if domainInfo.federal_type + else domainInfo.get_organization_type_display(), "Federal agency": domainInfo.federal_agency, "Organization name": domainInfo.organization_name, "City": domainInfo.city, From 1cb0127dfb66fe5662d08841f844c4b0be1bed58 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 11:14:18 -0400 Subject: [PATCH 29/38] tweak tests and linter --- src/registrar/tests/test_reports.py | 5 +++-- src/registrar/utility/csv_export.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 81380c929..828640956 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -39,6 +39,7 @@ class ExportDataTest(TestCase): domain=self.domain_1, organization_type="federal", federal_agency="World War I Centennial Commission", + federal_type="executive", ) self.domain_information_2, _ = DomainInformation.objects.get_or_create( creator=self.user, @@ -114,7 +115,7 @@ class ExportDataTest(TestCase): "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" "adomain2.gov,Interstate,dnsneeded\n" - "cdomain1.gov,Federal,World War I Centennial Commission,ready\n" + "cdomain1.gov,Federal - Executive,World War I Centennial Commission,ready\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,onhold\n" ) @@ -174,7 +175,7 @@ class ExportDataTest(TestCase): expected_content = ( "Domain name,Domain type,Federal agency,Organization name,City," "State,Security Contact Email\n" - "cdomain1.gov,Federal,World War I Centennial Commission\n" + "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" "ddomain3.gov,Federal,Armed Forces Retirement Home\n" ) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 339a42111..c23a6c8d9 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -20,7 +20,8 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): FIELDS = { "Domain name": domainInfo.domain.name, "Domain type": domainInfo.get_organization_type_display() - + " - " + domainInfo.federal_type + + " - " + + domainInfo.get_federal_type_display() if domainInfo.federal_type else domainInfo.get_organization_type_display(), "Federal agency": domainInfo.federal_agency, From ee49290e9811f4ca8221e3e83cb616bb975c23c4 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 13:02:34 -0400 Subject: [PATCH 30/38] CSS tweaks to the button layout in django admin (object-tools) --- src/registrar/admin.py | 2 + src/registrar/assets/sass/_theme/_admin.scss | 45 +++++++++++++++++++ .../django/admin/domain_change_list.html | 6 +-- src/registrar/tests/test_reports.py | 10 ++--- src/registrar/utility/csv_export.py | 8 ++-- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 8abf7a53b..8a2431fe1 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -778,6 +778,8 @@ class DomainAdmin(ListHeaderAdmin): urlpatterns = super().get_urls() + #Used to extrapolate a path name, for instance + # name="{app_label}_{model_name}_export_data_type" info = self.model._meta.app_label, self.model._meta.model_name my_url = [ diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 35d089cbd..6e2f6797e 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -180,3 +180,48 @@ h1, h2, h3 { background: var(--primary); color: var(--header-link-color); } + +// Font mismatch issue due to conflicts between django and uswds, +// rough overrides for consistency and readability. May want to revise +// in the future +.object-tools li a, +.object-tools p a { + font-family: "Source Sans Pro Web", "Helvetica Neue", Helvetica, Roboto, Arial, sans-serif; + text-transform: capitalize!important; + font-size: 14px!important; +} + +// For consistency, make the overrided p a +// object tool buttons the same size as the ul li a +.object-tools p { + line-height: 1.25rem; +} + +// Fix margins in mobile view +@media (max-width: 767px) { + .object-tools li { + // our CSS is read before django's, so need !important + // to override + margin-left: 0!important; + margin-right: 15px; + } +} + +// Fix height of buttons +.object-tools li { + height: auto; +} + +// Fixing height of buttons breaks layout because +// object-tools and changelist are siblings with +// flexbox positioning +#changelist { + clear: both; +} + +// Account for the h2, roughly 90px +@include at-media(tablet) { + .object-tools { + padding-left: 90px; + } +} \ No newline at end of file diff --git a/src/registrar/templates/django/admin/domain_change_list.html b/src/registrar/templates/django/admin/domain_change_list.html index b99f71e2f..68fdbe7aa 100644 --- a/src/registrar/templates/django/admin/domain_change_list.html +++ b/src/registrar/templates/django/admin/domain_change_list.html @@ -14,9 +14,9 @@ {% if has_add_permission %}
  • - - Add Domain - + + Add Domain +
    • {% endif %} diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 828640956..404ed358c 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -77,7 +77,7 @@ class ExportDataTest(TestCase): columns = [ "Domain name", "Domain type", - "Federal agency", + "Agency", "Organization name", "City", "State", @@ -111,7 +111,7 @@ class ExportDataTest(TestCase): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Domain type,Federal agency,Organization name,City,State,AO," + "Domain name,Domain type,Agency,Organization name,City,State,AO," "AO email,Submitter,Submitter title,Submitter email,Submitter phone," "Security Contact Email,Status\n" "adomain2.gov,Interstate,dnsneeded\n" @@ -134,7 +134,7 @@ class ExportDataTest(TestCase): self.assertEqual(csv_content, expected_content) - def test_export_domains_to_writer_2(self): + def test_export_domains_to_writer_additional(self): """An additional test for filters and multi-column sort""" # Create a CSV file in memory csv_file = StringIO() @@ -144,7 +144,7 @@ class ExportDataTest(TestCase): columns = [ "Domain name", "Domain type", - "Federal agency", + "Agency", "Organization name", "City", "State", @@ -173,7 +173,7 @@ class ExportDataTest(TestCase): # federal only # sorted alphabetially by domain name expected_content = ( - "Domain name,Domain type,Federal agency,Organization name,City," + "Domain name,Domain type,Agency,Organization name,City," "State,Security Contact Email\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" "ddomain3.gov,Federal,Armed Forces Retirement Home\n" diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index c23a6c8d9..ffada0a0b 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -24,7 +24,7 @@ def export_domains_to_writer(writer, columns, sort_fields, filter_condition): + domainInfo.get_federal_type_display() if domainInfo.federal_type else domainInfo.get_organization_type_display(), - "Federal agency": domainInfo.federal_agency, + "Agency": domainInfo.federal_agency, "Organization name": domainInfo.organization_name, "City": domainInfo.city, "State": domainInfo.state_territory, @@ -51,7 +51,7 @@ def export_data_type_to_csv(csv_file): columns = [ "Domain name", "Domain type", - "Federal agency", + "Agency", "Organization name", "City", "State", @@ -78,7 +78,7 @@ def export_data_full_to_csv(csv_file): columns = [ "Domain name", "Domain type", - "Federal agency", + "Agency", "Organization name", "City", "State", @@ -101,7 +101,7 @@ def export_data_federal_to_csv(csv_file): columns = [ "Domain name", "Domain type", - "Federal agency", + "Agency", "Organization name", "City", "State", From a1c351277e03b6f77aaf784d49449c1c0c18c9f3 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 13:48:07 -0400 Subject: [PATCH 31/38] lint --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 8a2431fe1..8914e5c87 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -778,7 +778,7 @@ class DomainAdmin(ListHeaderAdmin): urlpatterns = super().get_urls() - #Used to extrapolate a path name, for instance + # Used to extrapolate a path name, for instance # name="{app_label}_{model_name}_export_data_type" info = self.model._meta.app_label, self.model._meta.model_name From 0d504c742f788cf25106017c3697825e685d0d2d Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 31 Oct 2023 13:53:34 -0400 Subject: [PATCH 32/38] End of line on scss admin --- src/registrar/assets/sass/_theme/_admin.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 6e2f6797e..68ff51597 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -224,4 +224,4 @@ h1, h2, h3 { .object-tools { padding-left: 90px; } -} \ No newline at end of file +} From b05c337efe668ef18c9b3698dd275d58e2b6da63 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Wed, 1 Nov 2023 10:57:28 -0500 Subject: [PATCH 33/38] Configure a Login.gov production OIDC provider --- ops/manifests/manifest-stable.yaml | 2 ++ src/registrar/config/settings.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/ops/manifests/manifest-stable.yaml b/ops/manifests/manifest-stable.yaml index 6295fa63b..a47652326 100644 --- a/ops/manifests/manifest-stable.yaml +++ b/ops/manifests/manifest-stable.yaml @@ -23,6 +23,8 @@ applications: DJANGO_LOG_LEVEL: INFO # default public site location GETGOV_PUBLIC_SITE_URL: https://beta.get.gov + # Which OIDC provider to use + OIDC_ACTIVE_PROVIDER: login.gov production routes: - route: getgov-stable.app.cloud.gov services: diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 3e5734dcc..fd3642d72 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -49,6 +49,7 @@ env_debug = env.bool("DJANGO_DEBUG", default=False) env_log_level = env.str("DJANGO_LOG_LEVEL", "DEBUG") env_base_url = env.str("DJANGO_BASE_URL") env_getgov_public_site_url = env.str("GETGOV_PUBLIC_SITE_URL", "") +env_oidc_active_provider = env.str("OIDC_ACTIVE_PROVIDER", "identity sandbox") secret_login_key = b64decode(secret("DJANGO_SECRET_LOGIN_KEY", "")) secret_key = secret("DJANGO_SECRET_KEY") @@ -482,11 +483,12 @@ OIDC_ALLOW_DYNAMIC_OP = False # which provider to use if multiple are available # (code does not currently support user selection) -OIDC_ACTIVE_PROVIDER = "login.gov" +# See above for the default value if the env variable is missing +OIDC_ACTIVE_PROVIDER = env_oidc_active_provider OIDC_PROVIDERS = { - "login.gov": { + "identity sandbox": { "srv_discovery_url": "https://idp.int.identitysandbox.gov", "behaviour": { # the 'code' workflow requires direct connectivity from us to Login.gov @@ -502,6 +504,22 @@ OIDC_PROVIDERS = { "token_endpoint_auth_method": ["private_key_jwt"], "sp_private_key": secret_login_key, }, + "login.gov production": { + "srv_discovery_url": "https://secure.login.gov", + "behaviour": { + # the 'code' workflow requires direct connectivity from us to Login.gov + "response_type": "code", + "scope": ["email", "profile:name", "phone"], + "user_info_request": ["email", "first_name", "last_name", "phone"], + "acr_value": "http://idmanagement.gov/ns/assurance/ial/2", + }, + "client_registration": { + "client_id": "urn:gov:cisa:openidconnect.profiles:sp:sso:cisa:dotgov_registrar", + "redirect_uris": [f"{env_base_url}/openid/callback/login/"], + "post_logout_redirect_uris": [f"{env_base_url}/openid/callback/logout/"], + "token_endpoint_auth_method": ["private_key_jwt"], + "sp_private_key": secret_login_key, + }, } } From 14fd0b7baabc805e9437264b17e2cbf1ef49a551 Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Wed, 1 Nov 2023 11:36:33 -0500 Subject: [PATCH 34/38] drive-by instances increase --- ops/manifests/manifest-stable.yaml | 2 +- src/registrar/config/settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ops/manifests/manifest-stable.yaml b/ops/manifests/manifest-stable.yaml index a47652326..e7b3c74ae 100644 --- a/ops/manifests/manifest-stable.yaml +++ b/ops/manifests/manifest-stable.yaml @@ -4,7 +4,7 @@ applications: buildpacks: - python_buildpack path: ../../src - instances: 1 + instances: 2 memory: 512M stack: cflinuxfs4 timeout: 180 diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index fd3642d72..6b4d597a8 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -504,6 +504,7 @@ OIDC_PROVIDERS = { "token_endpoint_auth_method": ["private_key_jwt"], "sp_private_key": secret_login_key, }, + }, "login.gov production": { "srv_discovery_url": "https://secure.login.gov", "behaviour": { From f55014b8a8bcc5e65ee399a6ad57f7581942037b Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Wed, 1 Nov 2023 13:19:36 -0500 Subject: [PATCH 35/38] fix linting error --- src/registrar/config/settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 6b4d597a8..e0dedb60c 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -371,8 +371,7 @@ LOGGING = { # each handler has its choice of format "formatters": { "verbose": { - "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] " - "%(message)s", + "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", "datefmt": "%d/%b/%Y %H:%M:%S", }, "simple": { @@ -515,13 +514,15 @@ OIDC_PROVIDERS = { "acr_value": "http://idmanagement.gov/ns/assurance/ial/2", }, "client_registration": { - "client_id": "urn:gov:cisa:openidconnect.profiles:sp:sso:cisa:dotgov_registrar", + "client_id": ( + "urn:gov:cisa:openidconnect.profiles:sp:sso:cisa:dotgov_registrar" + ), "redirect_uris": [f"{env_base_url}/openid/callback/login/"], "post_logout_redirect_uris": [f"{env_base_url}/openid/callback/logout/"], "token_endpoint_auth_method": ["private_key_jwt"], "sp_private_key": secret_login_key, }, - } + }, } # endregion From d57e48a090af69a56a1f3097e8f6f860fafde60a Mon Sep 17 00:00:00 2001 From: Neil Martinsen-Burrell Date: Wed, 1 Nov 2023 13:28:52 -0500 Subject: [PATCH 36/38] Review feedback: use two instances on staging too --- ops/manifests/manifest-staging.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/manifests/manifest-staging.yaml b/ops/manifests/manifest-staging.yaml index 3e80352ba..a1d09a555 100644 --- a/ops/manifests/manifest-staging.yaml +++ b/ops/manifests/manifest-staging.yaml @@ -4,7 +4,7 @@ applications: buildpacks: - python_buildpack path: ../../src - instances: 1 + instances: 2 memory: 512M stack: cflinuxfs4 timeout: 180 From a0b405f713a9361e7b3fcca083e174204d277e73 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 2 Nov 2023 12:49:25 -0500 Subject: [PATCH 37/38] Scrubbed test files of any possible PII --- src/registrar/tests/data/test_contacts.txt | 14 +++++++------- .../tests/data/test_domain_contacts.txt | 16 ++++++++-------- .../tests/data/test_domain_statuses.txt | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/registrar/tests/data/test_contacts.txt b/src/registrar/tests/data/test_contacts.txt index dec8f6816..89f57ccf8 100644 --- a/src/registrar/tests/data/test_contacts.txt +++ b/src/registrar/tests/data/test_contacts.txt @@ -1,7 +1,7 @@ -TESTUSER|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||testuser@gmail.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| -RJD1|52545_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||agustina.wyman7@test.com|GSA|VERISIGN|ctldbatch|2021-06-29T18:53:09Z|VERISIGN|ctldbatch|2021-06-29T18:58:08Z| -JAKING|52555_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||susy.martin4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T15:38:10Z| -JBOONE|52556_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||stephania.winters4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-06-30T18:28:09Z| -MKELLEY|52557_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||alexandra.bobbitt5@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T15:23:10Z|VERISIGN|ctldbatch|2021-08-02T22:13:09Z| -CWILSON|52562_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||jospeh.mcdowell3@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:33:09Z| -LMCCADE|52563_CONTACT_GOV-VRSN|919-000-0000||918-000-0000||reginald.ratcliff4@test.com|GSA|VERISIGN|ctldbatch|2021-06-30T17:58:09Z|VERISIGN|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file +TESTUSER|12363_CONTACT|123-123-1234||918-000-0000||testuser@gmail.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| +USER1|12345_CONTACT|123-123-1234||918-000-0000||agustina.wyman7@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-29T18:53:09Z|SOMECOMPANY|ctldbatch|2021-06-29T18:58:08Z| +USER2|12355_CONTACT|123-123-1234||918-000-0000||susy.martin4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T15:38:10Z| +USER3|12356_CONTACT|123-123-1234||918-000-0000||stephania.winters4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-06-30T18:28:09Z| +USER4|12357_CONTACT|123-123-1234||918-000-0000||alexandra.bobbitt5@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T15:23:10Z|SOMECOMPANY|ctldbatch|2021-08-02T22:13:09Z| +USER5|12362_CONTACT|123-123-1234||918-000-0000||jospeh.mcdowell3@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:33:09Z| +USER6|12363_CONTACT|123-123-1234||918-000-0000||reginald.ratcliff4@test.com|GSA|SOMECOMPANY|ctldbatch|2021-06-30T17:58:09Z|SOMECOMPANY|ctldbatch|2021-06-30T18:18:09Z| \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_contacts.txt b/src/registrar/tests/data/test_domain_contacts.txt index 069e5231e..3a1ed745f 100644 --- a/src/registrar/tests/data/test_domain_contacts.txt +++ b/src/registrar/tests/data/test_domain_contacts.txt @@ -1,8 +1,8 @@ -Anomaly.gov|ANOMALY|tech -TestDomain.gov|TESTUSER|admin -NEHRP.GOV|RJD1|admin -NEHRP.GOV|JAKING|tech -NEHRP.GOV|JBOONE|billing -NELSONCOUNTY-VA.GOV|MKELLEY|admin -NELSONCOUNTY-VA.GOV|CWILSON|billing -NELSONCOUNTY-VA.GOV|LMCCADE|tech \ No newline at end of file +Anomaly.gov|ANOMALY|tech +TestDomain.gov|TESTUSER|admin +FakeWebsite1|USER1|admin +FakeWebsite1|USER2|tech +FakeWebsite1|USER3|billing +FakeWebsite2.GOV|USER4|admin +FakeWebsite2.GOV|USER5|billing +FakeWebsite2.GOV|USER6|tech \ No newline at end of file diff --git a/src/registrar/tests/data/test_domain_statuses.txt b/src/registrar/tests/data/test_domain_statuses.txt index 1f3cc8998..021e52ae7 100644 --- a/src/registrar/tests/data/test_domain_statuses.txt +++ b/src/registrar/tests/data/test_domain_statuses.txt @@ -1,4 +1,4 @@ Anomaly.gov|muahaha| -TestDomain.gov|ok| -NEHRP.GOV|serverHold| -NELSONCOUNTY-VA.GOV|Hold| \ No newline at end of file +TestDomain.gov|ok| +FakeWebsite1.GOV|serverHold| +FakeWebsite2.GOV|Hold| \ No newline at end of file From 183a4bf80156c13735d47eebc18efcbd6dcd413b Mon Sep 17 00:00:00 2001 From: CuriousX Date: Thu, 2 Nov 2023 12:58:50 -0500 Subject: [PATCH 38/38] Update data_migration.md --- docs/operations/data_migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index b577f062d..68a6419d5 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -97,10 +97,10 @@ Do not use these environments to store data you want to keep around permanently. #### STEP 1: Using cat to transfer data to sandboxes ```bash -cat {LOCAL_PATH_TO_FILE} | cf ssh {FULL_NAME_OF_YOUR_SANDBOX_HERE} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" +cat {LOCAL_PATH_TO_FILE} | cf ssh {APP_NAME_IN_ENVIRONMENT} -c "cat > /home/vcap/tmp/{DESIRED_NAME_OF_FILE}" ``` -* FULL_NAME_OF_YOUR_SANDBOX_HERE - Name of your sandbox, ex: getgov-za +* APP_NAME_IN_ENVIRONMENT - Name of your sandbox, ex: getgov-za * LOCAL_PATH_TO_FILE - Path to the file you want to copy, ex: src/tmp/escrow_contacts.daily.gov.GOV.txt * DESIRED_NAME_OF_FILE - Use this to specify the filename and type, ex: test.txt or escrow_contacts.daily.gov.GOV.txt