From d84a7890224ca1ec52c2e5097e8c0c1104589cdd Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 26 Dec 2024 17:10:37 -0800 Subject: [PATCH 001/201] Renewal form --- src/registrar/config/urls.py | 5 ++ src/registrar/fixtures/fixtures_domains.py | 2 +- src/registrar/models/domain.py | 5 ++ src/registrar/templates/domain_base.html | 9 +- src/registrar/templates/domain_detail.html | 4 +- src/registrar/templates/domain_sidebar.html | 10 ++- src/registrar/templatetags/custom_filters.py | 1 + src/registrar/views/__init__.py | 1 + src/registrar/views/domain.py | 92 ++++++++++++++++++-- 9 files changed, 117 insertions(+), 12 deletions(-) diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 66708c571..2bf7b9e5f 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -345,6 +345,11 @@ urlpatterns = [ views.DomainSecurityEmailView.as_view(), name="domain-security-email", ), + path( + "domain//renewal", + views.DomainRenewalView.as_view(), + name="domain-renewal", + ), path( "domain//users/add", views.DomainAddUserView.as_view(), diff --git a/src/registrar/fixtures/fixtures_domains.py b/src/registrar/fixtures/fixtures_domains.py index 4606024d0..4d4115180 100644 --- a/src/registrar/fixtures/fixtures_domains.py +++ b/src/registrar/fixtures/fixtures_domains.py @@ -44,7 +44,7 @@ class DomainFixture(DomainRequestFixture): cls._approve_domain_requests(users) @staticmethod - def _generate_fake_expiration_date(days_in_future=365): + def _generate_fake_expiration_date(days_in_future=40): """Generates a fake expiration date between 1 and 365 days in the future.""" current_date = timezone.now().date() # nosec return current_date + timedelta(days=random.randint(1, days_in_future)) # nosec diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 6eb2fac07..3fa6a61b2 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1167,6 +1167,11 @@ class Domain(TimeStampedModel, DomainHelper): threshold_date = now + timedelta(days=60) return now < self.expiration_date <= threshold_date + ###dummy method for testing for domain renewal form fail or success banner + + def update_expiration(self, success=True): + return success + def state_display(self, request=None): """Return the display status of the domain.""" if self.is_expired() and (self.state != self.State.UNKNOWN): diff --git a/src/registrar/templates/domain_base.html b/src/registrar/templates/domain_base.html index 9f7e8d2e6..de8e88791 100644 --- a/src/registrar/templates/domain_base.html +++ b/src/registrar/templates/domain_base.html @@ -1,5 +1,7 @@ {% extends "base.html" %} {% load static %} +{% load static url_helpers %} + {% block title %}{{ domain.name }} | {% endblock %} @@ -53,8 +55,11 @@ {% endif %} {% block domain_content %} - + {% if request.path|endswith:"renewal"%} +

Renew {{domain.name}}

+ {%else%}

Domain Overview

+ {% endif%} {% endblock %} {# domain_content #} {% endif %} @@ -62,4 +67,4 @@ -{% endblock %} {# content #} +{% endblock %} {# content #} \ No newline at end of file diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index a5b8e52cb..b168f7e82 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -50,7 +50,9 @@ {% if domain.get_state_help_text %}
{% if has_domain_renewal_flag and domain.is_expiring and is_domain_manager %} - This domain will expire soon. Renew to maintain access. + This domain will expire soon. + {% url 'domain-renewal' pk=domain.id as url %} + Renew to maintain access. {% elif has_domain_renewal_flag and domain.is_expiring and is_portfolio_user %} This domain will expire soon. Contact one of the listed domain managers to renew the domain. {% else %} diff --git a/src/registrar/templates/domain_sidebar.html b/src/registrar/templates/domain_sidebar.html index 289f544ce..dc97f5ca1 100644 --- a/src/registrar/templates/domain_sidebar.html +++ b/src/registrar/templates/domain_sidebar.html @@ -79,8 +79,14 @@ {% with url_name="domain-users" %} {% include "includes/domain_sidenav_item.html" with item_text="Domain managers" %} {% endwith %} - + + {% if has_domain_renewal_flag and is_domain_manager and domain.is_expiring %} + {% with url_name="domain-renewal" %} + {% include "includes/domain_sidenav_item.html" with item_text="Renewal form" %} + {% endwith %} + {% endif %} + {% endif %} -
+ \ No newline at end of file diff --git a/src/registrar/templatetags/custom_filters.py b/src/registrar/templatetags/custom_filters.py index 6140130c8..6f3894ea5 100644 --- a/src/registrar/templatetags/custom_filters.py +++ b/src/registrar/templatetags/custom_filters.py @@ -200,6 +200,7 @@ def is_domain_subpage(path): "domain-users-add", "domain-request-delete", "domain-user-delete", + "domain-renewal", "invitation-cancel", ] return get_url_name(path) in url_names diff --git a/src/registrar/views/__init__.py b/src/registrar/views/__init__.py index a80b16b1a..4e3faced1 100644 --- a/src/registrar/views/__init__.py +++ b/src/registrar/views/__init__.py @@ -14,6 +14,7 @@ from .domain import ( DomainInvitationCancelView, DomainDeleteUserView, PrototypeDomainDNSRecordView, + DomainRenewalView, ) from .user_profile import UserProfileView, FinishProfileSetupView from .health import * diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index cb3da1f83..99a173517 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -12,7 +12,7 @@ from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError from django.http import HttpResponseRedirect -from django.shortcuts import redirect +from django.shortcuts import redirect, render from django.urls import reverse from django.views.generic.edit import FormMixin from django.conf import settings @@ -307,6 +307,90 @@ class DomainView(DomainBaseView): self._update_session_with_domain() +class DomainRenewalView(DomainBaseView): + """Domain detail overview page.""" + + template_name = "domain_renewal.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + default_emails = [DefaultEmail.PUBLIC_CONTACT_DEFAULT.value, DefaultEmail.LEGACY_DEFAULT.value] + context["hidden_security_emails"] = default_emails + + security_email = self.object.get_security_email() + user = self.request.user + if security_email is None or security_email in default_emails: + context["security_email"] = None + context["user"] = user + return context + + def can_access_domain_via_portfolio(self, pk): + """Most views should not allow permission to portfolio users. + If particular views allow permissions, they will need to override + this function.""" + portfolio = self.request.session.get("portfolio") + if self.request.user.has_any_domains_portfolio_permission(portfolio): + if Domain.objects.filter(id=pk).exists(): + domain = Domain.objects.get(id=pk) + if domain.domain_info.portfolio == portfolio: + return True + return False + + def in_editable_state(self, pk): + """Override in_editable_state from DomainPermission + Allow detail page to be viewable""" + + requested_domain = None + if Domain.objects.filter(id=pk).exists(): + requested_domain = Domain.objects.get(id=pk) + + # return true if the domain exists, this will allow the detail page to load + if requested_domain: + return True + return False + + def _get_domain(self, request): + """ + override get_domain for this view so that domain overview + always resets the cache for the domain object + """ + self.session = request.session + self.object = self.get_object() + self._update_session_with_domain() + + def post(self, request, pk): + domain = Domain.objects.filter(id=pk).first() + + # Check if the checkbox is checked + is_policy_acknowledged = request.POST.get("is_policy_acknowledged", None) + if is_policy_acknowledged != "on": + print("!!! Checkbox is NOT acknowledged") + messages.error( + request, "Check the box if you read and agree to the requirements for operating a .gov domain." + ) + return render( + request, + "domain_renewal.html", + { + "domain": domain, + "form": request.POST, + }, + ) + + print("*** Checkbox is acknowledged") + if "submit_button" in request.POST: + print("*** Submit button clicked") + updated_expiration = domain.update_expiration(success=True) + print("*** Updated expiration result:", updated_expiration) + + if updated_expiration is True: + messages.success(request, "This domain has been renewed for one year") + else: + messages.error(request, "This domain has not been renewed") + return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) + + class DomainOrgNameAddressView(DomainFormBaseView): """Organization view""" @@ -807,11 +891,7 @@ class DomainDNSSECView(DomainFormBaseView): has_dnssec_records = self.object.dnssecdata is not None # Create HTML for the modal button - modal_button = ( - '' - ) + modal_button = '' context["modal_button"] = modal_button context["has_dnssec_records"] = has_dnssec_records From b5658a357b6aa9b49c7c37393ae1f8893dbebbeb Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Thu, 26 Dec 2024 17:15:07 -0800 Subject: [PATCH 002/201] Fix linter --- src/registrar/models/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 3fa6a61b2..715f1b9da 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1167,7 +1167,7 @@ class Domain(TimeStampedModel, DomainHelper): threshold_date = now + timedelta(days=60) return now < self.expiration_date <= threshold_date - ###dummy method for testing for domain renewal form fail or success banner + # Dummy method for testing for domain renewal form fail or success banner def update_expiration(self, success=True): return success From 8b9eb93b682a3f8b714c42558b90db6b94bc7c57 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Mon, 30 Dec 2024 11:07:33 -0500 Subject: [PATCH 003/201] added back domain renewal form --- src/registrar/templates/domain_renewal.html | 127 ++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/registrar/templates/domain_renewal.html diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html new file mode 100644 index 000000000..c179941d6 --- /dev/null +++ b/src/registrar/templates/domain_renewal.html @@ -0,0 +1,127 @@ +{% extends "domain_base.html" %} +{% load static url_helpers %} +{% load custom_filters %} + +{% block domain_content %} + {% block breadcrumb %} + {% if portfolio %} + + + {% endif %} + {% endblock breadcrumb %} + + {{ block.super }} +
+

Confirm the following information for accuracy

+

Review these details below. We + require that you maintain accurate information for the domain. + The details you provide will only be used to support eh administration of .gov and won't be made public. +

+

If you would like to retire your domain instead, please + contact us.

+

Required fields are marked with an asterisk (*). +

+
+ + {% url 'user-profile' as url %} + {% include "includes/summary_item.html" with title='Your Contact Information' value=user edit_link=url editable=is_editable contact='true' %} + + + + {% if analyst_action != 'edit' or analyst_action_location != domain.pk %} + {% if is_portfolio_user and not is_domain_manager %} +
+
+

+ You don't have access to manage {{domain.name}}. If you need to make updates, contact one of the listed domain managers. +

+
+
+ {% endif %} + {% endif %} + + {% url 'domain-security-email' pk=domain.id as url %} + {% if security_email is not None and security_email not in hidden_security_emails%} + {% include "includes/summary_item.html" with title='Security email' value=security_email edit_link=url editable=is_editable %} + {% else %} + {% include "includes/summary_item.html" with title='Security email' value='None provided' edit_link=url editable=is_editable %} + {% endif %} + {% url 'domain-users' pk=domain.id as url %} + {% if portfolio %} + {% include "includes/summary_item.html" with title='Domain managers' domain_permissions=True value=domain edit_link=url editable=is_editable %} + {% else %} + {% include "includes/summary_item.html" with title='Domain managers' list=True users=True value=domain.permissions.all edit_link=url editable=is_editable %} + {% endif %} + + +
+ +
+ +

+ Acknowledgement of .gov domain requirements +

+ + + {% if messages %} +
    + {% for message in messages %} +

    {{ message }}

    + {% endfor %} + + {% endif %} + + +
    + {% csrf_token %} +
    +
    + + + + + +{% endblock %} {# domain_content #} \ No newline at end of file From 19e5700a4db2dd3d46ab4490ec37911c0c678f14 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Mon, 30 Dec 2024 09:05:40 -0800 Subject: [PATCH 004/201] Add in the domain renewal template oops --- src/registrar/templates/domain_renewal.html | 127 ++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/registrar/templates/domain_renewal.html diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html new file mode 100644 index 000000000..eda7a8bdd --- /dev/null +++ b/src/registrar/templates/domain_renewal.html @@ -0,0 +1,127 @@ +{% extends "domain_base.html" %} +{% load static url_helpers %} +{% load custom_filters %} + +{% block domain_content %} + {% block breadcrumb %} + {% if portfolio %} + + + {% endif %} + {% endblock breadcrumb %} + + {{ block.super }} +
    +

    Confirm the following information for accuracy

    +

    Review these details below. We + require that you maintain accurate information for the domain. + The details you provide will only be used to support eh administration of .gov and won't be made public. +

    +

    If you would like to retire your domain instead, please + contact us.

    +

    Required fields are marked with an asterisk (*). +

    +
    + + {% url 'user-profile' as url %} + {% include "includes/summary_item.html" with title='Your Contact Information' value=user edit_link=url editable=is_editable contact='true' %} + + + + {% if analyst_action != 'edit' or analyst_action_location != domain.pk %} + {% if is_portfolio_user and not is_domain_manager %} +
    +
    +

    + You don't have access to manage {{domain.name}}. If you need to make updates, contact one of the listed domain managers. +

    +
    +
    + {% endif %} + {% endif %} + + {% url 'domain-security-email' pk=domain.id as url %} + {% if security_email is not None and security_email not in hidden_security_emails%} + {% include "includes/summary_item.html" with title='Security email' value=security_email edit_link=url editable=is_editable %} + {% else %} + {% include "includes/summary_item.html" with title='Security email' value='None provided' edit_link=url editable=is_editable %} + {% endif %} + {% url 'domain-users' pk=domain.id as url %} + {% if portfolio %} + {% include "includes/summary_item.html" with title='Domain managers' domain_permissions=True value=domain edit_link=url editable=is_editable %} + {% else %} + {% include "includes/summary_item.html" with title='Domain managers' list=True users=True value=domain.permissions.all edit_link=url editable=is_editable %} + {% endif %} + + +
    + +
    + +

    + Acknowledgement of .gov domain requirements +

    +     + + {% if messages %} +
      + {% for message in messages %} +

      {{ message }}

      + {% endfor %} + + {% endif %} + + +
      + {% csrf_token %} +
      +
      + + + + + +{% endblock %} {# domain_content #} \ No newline at end of file From e23d81f7e7d4096e76450b0cca84908846484872 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Mon, 30 Dec 2024 12:44:52 -0500 Subject: [PATCH 005/201] added renew method --- src/registrar/models/domain.py | 6 +++++- src/registrar/views/domain.py | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 715f1b9da..e170c8668 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -322,28 +322,32 @@ class Domain(TimeStampedModel, DomainHelper): """ # If no date is specified, grab the registry_expiration_date + print("checking if there is a date") try: exp_date = self.registry_expiration_date except KeyError: # if no expiration date from registry, set it to today logger.warning("current expiration date not set; setting to today") exp_date = date.today() - + print("we got the date", exp_date) # create RenewDomain request request = commands.RenewDomain(name=self.name, cur_exp_date=exp_date, period=epp.Period(length, unit)) try: # update expiration date in registry, and set the updated # expiration date in the registrar, and in the cache + print("we are in the second try") self._cache["ex_date"] = registry.send(request, cleaned=True).res_data[0].ex_date self.expiration_date = self._cache["ex_date"] self.save() except RegistryError as err: # if registry error occurs, log the error, and raise it as well + print("registry error") logger.error(f"registry error renewing domain: {err}") raise (err) except Exception as e: # exception raised during the save to registrar + print("this is the last error") logger.error(f"error updating expiration date in registrar: {e}") raise (e) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 99a173517..1c1996c65 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -381,12 +381,13 @@ class DomainRenewalView(DomainBaseView): print("*** Checkbox is acknowledged") if "submit_button" in request.POST: print("*** Submit button clicked") - updated_expiration = domain.update_expiration(success=True) - print("*** Updated expiration result:", updated_expiration) - - if updated_expiration is True: + # updated_expiration = domain.update_expiration(success=True) + # print("*** Updated expiration result:", updated_expiration) + try: + domain.renew_domain() messages.success(request, "This domain has been renewed for one year") - else: + except Exception as e: + print(f'An error occured: {e}') messages.error(request, "This domain has not been renewed") return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) From a128086d6f5e2cf51dda942dfc92c7dceb92a45d Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 31 Dec 2024 10:13:53 -0800 Subject: [PATCH 006/201] Update print statements and exp timing --- src/registrar/models/domain.py | 16 +++++++++------- src/registrar/views/domain.py | 5 +++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index e170c8668..b7145ec0c 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -322,32 +322,34 @@ class Domain(TimeStampedModel, DomainHelper): """ # If no date is specified, grab the registry_expiration_date - print("checking if there is a date") + print("*** Checking if there is a date") try: exp_date = self.registry_expiration_date except KeyError: # if no expiration date from registry, set it to today - logger.warning("current expiration date not set; setting to today") - exp_date = date.today() - print("we got the date", exp_date) + logger.warning("*** Current expiration date not set; setting to 35 days ") + # exp_date = date.today() + exp_date = date.today() - timedelta(days=35) + print(exp_date) + print("*** The exp_date is", exp_date) # create RenewDomain request request = commands.RenewDomain(name=self.name, cur_exp_date=exp_date, period=epp.Period(length, unit)) try: # update expiration date in registry, and set the updated # expiration date in the registrar, and in the cache - print("we are in the second try") + print("** In renew_domain in 2nd try statement") self._cache["ex_date"] = registry.send(request, cleaned=True).res_data[0].ex_date self.expiration_date = self._cache["ex_date"] self.save() except RegistryError as err: # if registry error occurs, log the error, and raise it as well - print("registry error") + print("*** Registry error") logger.error(f"registry error renewing domain: {err}") raise (err) except Exception as e: # exception raised during the save to registrar - print("this is the last error") + print("*** In renew_domain, in the last Exception statement") logger.error(f"error updating expiration date in registrar: {e}") raise (e) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 1c1996c65..dd8e9928b 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -384,11 +384,12 @@ class DomainRenewalView(DomainBaseView): # updated_expiration = domain.update_expiration(success=True) # print("*** Updated expiration result:", updated_expiration) try: + print("*** Did we get into the try statement") domain.renew_domain() messages.success(request, "This domain has been renewed for one year") except Exception as e: - print(f'An error occured: {e}') - messages.error(request, "This domain has not been renewed") + print(f"An error occured: {e}") + messages.error(request, "*** This domain has not been renewed") return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) From c145ecbfa29d925af2806343eb8d8534677d8ac5 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Tue, 31 Dec 2024 13:55:18 -0500 Subject: [PATCH 007/201] put back the fixtures command --- src/registrar/fixtures/fixtures_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/fixtures/fixtures_domains.py b/src/registrar/fixtures/fixtures_domains.py index 4d4115180..4606024d0 100644 --- a/src/registrar/fixtures/fixtures_domains.py +++ b/src/registrar/fixtures/fixtures_domains.py @@ -44,7 +44,7 @@ class DomainFixture(DomainRequestFixture): cls._approve_domain_requests(users) @staticmethod - def _generate_fake_expiration_date(days_in_future=40): + def _generate_fake_expiration_date(days_in_future=365): """Generates a fake expiration date between 1 and 365 days in the future.""" current_date = timezone.now().date() # nosec return current_date + timedelta(days=random.randint(1, days_in_future)) # nosec From 2651cd4abaebbe84c3ed762ce3bf99f28ab86122 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 2 Jan 2025 16:36:36 -0500 Subject: [PATCH 008/201] initial approach, from domain.py, not quite working --- src/registrar/forms/domain_request_wizard.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 289b3da0b..45bd575f9 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -296,6 +296,14 @@ class OrganizationContactForm(RegistrarForm): label="Urbanization (required for Puerto Rico only)", ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Set initial value for federal agency combo box and specify combobox template + if self.domain_request and self.domain_request.federal_agency: + self.fields["federal_agency"].initial = self.domain_request.federal_agency + self.fields["federal_agency"].widget.attrs["data-default-value"] = self.domain_request.federal_agency.pk + self.fields["federal_agency"].widget.template_name = "django/forms/widgets/combobox.html", + def clean_federal_agency(self): """Require something to be selected when this is a federal agency.""" federal_agency = self.cleaned_data.get("federal_agency", None) From e5f9696bac942a946c92430ff36dc6eba489b608 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Fri, 3 Jan 2025 10:43:35 -0500 Subject: [PATCH 009/201] added tests --- src/registrar/tests/test_views_domain.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index ba237e1e7..767445810 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -529,6 +529,21 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): ) self.assertContains(detail_page, "Renew to maintain access") + @override_flag("domain_renewal", active=True) + def test_domain_renewal_sidebar_and_form(self): + self.client.force_login(self.user) + with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( + Domain, "is_expired", self.custom_is_expired + ): + detail_page = self.client.get( + reverse("domain", kwargs={"pk": self.expiringdomain.id}), + ) + self.assertContains(detail_page, "Renewal form") + response = self.client.get(reverse("domain-renewal",kwargs={"pk": self.expiringdomain.id})) + self.assertEqual(response.status_code, 200) + self.assertContains(response, f"Renew {self.expiringdomain.name}") + + class TestDomainManagers(TestDomainOverview): @classmethod From 4feb29b269a39417e35fae2589b7026fbee757c3 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 3 Jan 2025 10:47:42 -0800 Subject: [PATCH 010/201] Add in tests for making edit is clickable and goes to right route --- src/registrar/tests/test_views_domain.py | 57 +++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 767445810..281b7291e 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -539,10 +539,63 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): reverse("domain", kwargs={"pk": self.expiringdomain.id}), ) self.assertContains(detail_page, "Renewal form") - response = self.client.get(reverse("domain-renewal",kwargs={"pk": self.expiringdomain.id})) + response = self.client.get(reverse("domain-renewal", kwargs={"pk": self.expiringdomain.id})) self.assertEqual(response.status_code, 200) self.assertContains(response, f"Renew {self.expiringdomain.name}") - + + @override_flag("domain_renewal", active=True) + def test_domain_renewal_form_your_contact_info_edit(self): + with less_console_noise(): + # Start on the Renewal page for the domain + renewal_page = self.app.get(reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id})) + + # Verify we see "Your Contact Information" on the renewal form + self.assertContains(renewal_page, "Your Contact Information") + + # Verify that the "Edit" button for Your Contact is there and links to correct URL + edit_button_url = reverse("user-profile") + self.assertContains(renewal_page, f'href="{edit_button_url}"') + + # Simulate clicking on edit button + edit_page = renewal_page.click(href=edit_button_url, index=1) + self.assertEqual(edit_page.status_code, 200) + self.assertContains(edit_page, "Review the details below and update any required information") + + @override_flag("domain_renewal", active=True) + def test_domain_renewal_form_security_contact_edit(self): + with less_console_noise(): + # Start on the Renewal page for the domain + renewal_page = self.app.get(reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id})) + + # Verify we see "Security email" on the renewal form + self.assertContains(renewal_page, "Security email") + + # Verify that the "Edit" button for Security email is there and links to correct URL + edit_button_url = reverse("domain-security-email", kwargs={"pk": self.domain_with_ip.id}) + self.assertContains(renewal_page, f'href="{edit_button_url}"') + + # Simulate clicking on edit button + edit_page = renewal_page.click(href=edit_button_url, index=1) + self.assertEqual(edit_page.status_code, 200) + self.assertContains(edit_page, "A security contact should be capable of evaluating") + + @override_flag("domain_renewal", active=True) + def test_domain_renewal_form_domain_manager_edit(self): + with less_console_noise(): + # Start on the Renewal page for the domain + renewal_page = self.app.get(reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id})) + + # Verify we see "Domain managers" on the renewal form + self.assertContains(renewal_page, "Domain managers") + + # Verify that the "Edit" button for Domain managers is there and links to correct URL + edit_button_url = reverse("domain-users", kwargs={"pk": self.domain_with_ip.id}) + self.assertContains(renewal_page, f'href="{edit_button_url}"') + + # Simulate clicking on edit button + edit_page = renewal_page.click(href=edit_button_url, index=1) + self.assertEqual(edit_page.status_code, 200) + self.assertContains(edit_page, "Domain managers can update all information related to a domain") class TestDomainManagers(TestDomainOverview): From 7938bbe8e2f3983e04d2e6708904f018dfecb17e Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 3 Jan 2025 12:26:46 -0700 Subject: [PATCH 011/201] Combine action column and delete columns. Make "Action" header visible --- .../src/js/getgov/table-domain-requests.js | 20 ++++++++++--------- .../includes/domain_requests_table.html | 2 +- .../templates/includes/domains_table.html | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/table-domain-requests.js b/src/registrar/assets/src/js/getgov/table-domain-requests.js index 51e4ea12b..3078a0fbf 100644 --- a/src/registrar/assets/src/js/getgov/table-domain-requests.js +++ b/src/registrar/assets/src/js/getgov/table-domain-requests.js @@ -98,7 +98,7 @@ export class DomainRequestsTable extends BaseTable { } if (request.is_deletable) { - // 1st path: Just a modal trigger in any screen size for non-org users + // 1st path (non-org): Just a modal trigger in any screen size for non-org users modalTrigger = ` - - - ${actionLabel} ${request.requested_domain ? request.requested_domain : 'New domain request'} - +
      + + + ${actionLabel} ${request.requested_domain ? request.requested_domain : 'New domain request'} + + ${customTableOptions.needsAdditionalColumn ? modalTrigger : ''} +
      - ${customTableOptions.needsAdditionalColumn ? ''+modalTrigger+'' : ''} `; tbody.appendChild(row); if (request.is_deletable) DomainRequestsTable.addDomainRequestsModal(request.requested_domain, request.id, request.created_at, tbody); diff --git a/src/registrar/templates/includes/domain_requests_table.html b/src/registrar/templates/includes/domain_requests_table.html index 56cdc2cec..9ab65ef7b 100644 --- a/src/registrar/templates/includes/domain_requests_table.html +++ b/src/registrar/templates/includes/domain_requests_table.html @@ -186,7 +186,7 @@ Created by {% endif %} Status - Action + Action diff --git a/src/registrar/templates/includes/domains_table.html b/src/registrar/templates/includes/domains_table.html index 15becea7a..db9f10087 100644 --- a/src/registrar/templates/includes/domains_table.html +++ b/src/registrar/templates/includes/domains_table.html @@ -214,7 +214,7 @@ scope="col" role="columnheader" > - Action + Action From 3f3ba22a119db0568861440f88dc0632d2ae4bfd Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 3 Jan 2025 11:28:48 -0800 Subject: [PATCH 012/201] Update text to click on link --- src/registrar/tests/test_views_domain.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 281b7291e..02d7aa9ac 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -535,11 +535,24 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( Domain, "is_expired", self.custom_is_expired ): + # Grab the detail page detail_page = self.client.get( reverse("domain", kwargs={"pk": self.expiringdomain.id}), ) + + # Make sure we see the link as a domain manager + self.assertContains(detail_page, "Renew to maintain access") + + # Make sure we can see Renewal form on the sidebar since it's expiring self.assertContains(detail_page, "Renewal form") - response = self.client.get(reverse("domain-renewal", kwargs={"pk": self.expiringdomain.id})) + + # Grab link to the renewal page + renewal_form_url = reverse("domain-renewal", kwargs={"pk": self.expiringdomain.id}) + self.assertContains(detail_page, f'href="{renewal_form_url}"') + + # Simulate clicking the link + response = self.client.get(renewal_form_url) + self.assertEqual(response.status_code, 200) self.assertContains(response, f"Renew {self.expiringdomain.name}") From dca70f4195237db300508421ecc59b0a43bab908 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Sat, 4 Jan 2025 14:17:55 -0800 Subject: [PATCH 013/201] Unit test for submit and recieving error message --- src/registrar/tests/test_views_domain.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 02d7aa9ac..50b8f31f6 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -9,6 +9,7 @@ from api.tests.common import less_console_noise_decorator from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from .common import MockEppLib, MockSESClient, create_user # type: ignore from django_webtest import WebTest # type: ignore +from django.contrib.messages import get_messages import boto3_mocking # type: ignore from registrar.utility.errors import ( @@ -610,6 +611,25 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): self.assertEqual(edit_page.status_code, 200) self.assertContains(edit_page, "Domain managers can update all information related to a domain") + @override_flag("domain_renewal", active=True) + def test_ack_checkbox_not_checked(self): + + # Grab the renewal URL + renewal_url = reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id}) + + # Test clicking the checkbox + response = self.client.post(renewal_url, data={"submit_button": "next"}) + + # Verify the error message is displayed + # Retrieves messages obj (used in the post call) + messages = list(get_messages(response.wsgi_request)) + # Check we only get 1 error message + self.assertEqual(len(messages), 1) + # Check that the 1 error msg also is the right text + self.assertEqual( + str(messages[0]), + "Check the box if you read and agree to the requirements for operating a .gov domain.", + ) class TestDomainManagers(TestDomainOverview): @classmethod From bad8a8c98a75c1df161321c6a49d4b087d19e291 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Sat, 4 Jan 2025 14:48:21 -0800 Subject: [PATCH 014/201] Fix padding on form and addin subtext for security email --- src/registrar/templates/domain_renewal.html | 7 ++++--- src/registrar/templates/includes/summary_item.html | 8 +++++--- src/registrar/tests/test_views_domain.py | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html index eda7a8bdd..4bf69dbca 100644 --- a/src/registrar/templates/domain_renewal.html +++ b/src/registrar/templates/domain_renewal.html @@ -26,6 +26,7 @@ {{ block.super }}

      Confirm the following information for accuracy

      +

      HELLO

      Review these details below. We require that you maintain accurate information for the domain. The details you provide will only be used to support eh administration of .gov and won't be made public. @@ -55,9 +56,9 @@ {% url 'domain-security-email' pk=domain.id as url %} {% if security_email is not None and security_email not in hidden_security_emails%} - {% include "includes/summary_item.html" with title='Security email' value=security_email edit_link=url editable=is_editable %} + {% include "includes/summary_item.html" with title='Security email' value=security_email custom_text_for_value_none='We strongly recommend that you provide a security email. This email will allow the public to report observed or suspected security issues on your domain.' edit_link=url editable=is_editable %} {% else %} - {% include "includes/summary_item.html" with title='Security email' value='None provided' edit_link=url editable=is_editable %} + {% include "includes/summary_item.html" with title='Security email' value='None provided' custom_text_for_value_none='We strongly recommend that you provide a security email. This email will allow the public to report observed or suspected security issues on your domain.' edit_link=url editable=is_editable %} {% endif %} {% url 'domain-users' pk=domain.id as url %} {% if portfolio %} @@ -67,7 +68,7 @@ {% endif %} -

      +
      diff --git a/src/registrar/templates/includes/summary_item.html b/src/registrar/templates/includes/summary_item.html index 15cc0f67f..facc8956a 100644 --- a/src/registrar/templates/includes/summary_item.html +++ b/src/registrar/templates/includes/summary_item.html @@ -128,11 +128,13 @@ {% endif %} {% else %}

      + {% if custom_text_for_value_none %} +

      {{ custom_text_for_value_none }}

      + {% endif %} {% if value %} {{ value }} - {% elif custom_text_for_value_none %} - {{ custom_text_for_value_none }} - {% else %} + {% endif %} + {% if not value and not custom_text_for_value_none %} None {% endif %}

      diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 50b8f31f6..fa4351de1 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -631,6 +631,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): "Check the box if you read and agree to the requirements for operating a .gov domain.", ) + class TestDomainManagers(TestDomainOverview): @classmethod def setUpClass(cls): From a3c50043c5a1d6fdc7154ad2944926d39181d9ac Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Sat, 4 Jan 2025 14:55:40 -0800 Subject: [PATCH 015/201] Removing print statements --- src/registrar/models/domain.py | 16 ++-------------- src/registrar/templates/domain_renewal.html | 1 - src/registrar/views/domain.py | 7 ------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index b7145ec0c..59e04e7c3 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -322,34 +322,27 @@ class Domain(TimeStampedModel, DomainHelper): """ # If no date is specified, grab the registry_expiration_date - print("*** Checking if there is a date") try: exp_date = self.registry_expiration_date except KeyError: # if no expiration date from registry, set it to today - logger.warning("*** Current expiration date not set; setting to 35 days ") - # exp_date = date.today() - exp_date = date.today() - timedelta(days=35) - print(exp_date) - print("*** The exp_date is", exp_date) + logger.warning("current expiration date not set; setting to today") + exp_date = date.today() # create RenewDomain request request = commands.RenewDomain(name=self.name, cur_exp_date=exp_date, period=epp.Period(length, unit)) try: # update expiration date in registry, and set the updated # expiration date in the registrar, and in the cache - print("** In renew_domain in 2nd try statement") self._cache["ex_date"] = registry.send(request, cleaned=True).res_data[0].ex_date self.expiration_date = self._cache["ex_date"] self.save() except RegistryError as err: # if registry error occurs, log the error, and raise it as well - print("*** Registry error") logger.error(f"registry error renewing domain: {err}") raise (err) except Exception as e: # exception raised during the save to registrar - print("*** In renew_domain, in the last Exception statement") logger.error(f"error updating expiration date in registrar: {e}") raise (e) @@ -1173,11 +1166,6 @@ class Domain(TimeStampedModel, DomainHelper): threshold_date = now + timedelta(days=60) return now < self.expiration_date <= threshold_date - # Dummy method for testing for domain renewal form fail or success banner - - def update_expiration(self, success=True): - return success - def state_display(self, request=None): """Return the display status of the domain.""" if self.is_expired() and (self.state != self.State.UNKNOWN): diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html index 4bf69dbca..07aeecd1a 100644 --- a/src/registrar/templates/domain_renewal.html +++ b/src/registrar/templates/domain_renewal.html @@ -26,7 +26,6 @@ {{ block.super }}

      Confirm the following information for accuracy

      -

      HELLO

      Review these details below. We require that you maintain accurate information for the domain. The details you provide will only be used to support eh administration of .gov and won't be made public. diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 44ac9e9d6..ab68addb7 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -365,7 +365,6 @@ class DomainRenewalView(DomainBaseView): # Check if the checkbox is checked is_policy_acknowledged = request.POST.get("is_policy_acknowledged", None) if is_policy_acknowledged != "on": - print("!!! Checkbox is NOT acknowledged") messages.error( request, "Check the box if you read and agree to the requirements for operating a .gov domain." ) @@ -378,17 +377,11 @@ class DomainRenewalView(DomainBaseView): }, ) - print("*** Checkbox is acknowledged") if "submit_button" in request.POST: - print("*** Submit button clicked") - # updated_expiration = domain.update_expiration(success=True) - # print("*** Updated expiration result:", updated_expiration) try: - print("*** Did we get into the try statement") domain.renew_domain() messages.success(request, "This domain has been renewed for one year") except Exception as e: - print(f"An error occured: {e}") messages.error(request, "*** This domain has not been renewed") return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) From 0118e1f00dd2abe550d851c5fb0ba77f81b0149e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:36:44 -0700 Subject: [PATCH 016/201] Add suborg data and clean duplicate script --- .../commands/clean_duplicate_suborgs.py | 123 ++++++++++++++++++ .../commands/create_federal_portfolio.py | 23 +++- 2 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 src/registrar/management/commands/clean_duplicate_suborgs.py diff --git a/src/registrar/management/commands/clean_duplicate_suborgs.py b/src/registrar/management/commands/clean_duplicate_suborgs.py new file mode 100644 index 000000000..a5c7a87f0 --- /dev/null +++ b/src/registrar/management/commands/clean_duplicate_suborgs.py @@ -0,0 +1,123 @@ +import logging +from django.core.management import BaseCommand +from registrar.models import Suborganization, DomainRequest, DomainInformation +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" + + def handle(self, **kwargs): + # Find duplicates + duplicates = {} + all_suborgs = Suborganization.objects.all() + + for suborg in all_suborgs: + # Normalize name by removing extra spaces and converting to lowercase + normalized_name = " ".join(suborg.name.split()).lower() + + # First occurrence of this name + if normalized_name not in duplicates: + duplicates[normalized_name] = { + "keep": suborg, + "delete": [] + } + continue + + # Compare with our current best + current_best = duplicates[normalized_name]["keep"] + + # Check if all other fields match. + # If they don't, we should inspect this record manually. + fields_to_compare = ["portfolio", "city", "state_territory"] + fields_match = all( + getattr(suborg, field) == getattr(current_best, field) + for field in fields_to_compare + ) + if not fields_match: + logger.warning( + f"{TerminalColors.YELLOW}" + f"\nSkipping potential duplicate: {suborg.name} (id: {suborg.id})" + f"\nData mismatch with {current_best.name} (id: {current_best.id})" + f"{TerminalColors.ENDC}" + ) + continue + + # Determine if new suborg is better than current best. + # The fewest spaces and most capitals wins. + new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") + new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) + # TODO + # Split into words and count properly capitalized first letters + # new_proper_caps = sum( + # 1 for word in suborg.name.split() + # if word and word[0].isupper() + # ) + # current_proper_caps = sum( + # 1 for word in current_best.name.split() + # if word and word[0].isupper() + # ) + # new_has_better_caps = new_proper_caps > current_proper_caps + + if new_has_fewer_spaces or new_has_more_capitals: + # New suborg is better - demote the old one to the delete list + duplicates[normalized_name]["delete"].append(current_best) + duplicates[normalized_name]["keep"] = suborg + else: + # If it is not better, just delete the old one + duplicates[normalized_name]["delete"].append(suborg) + + # Filter out entries without duplicates + duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} + if not duplicates: + logger.info(f"No duplicate suborganizations found.") + return + + # Show preview of changes + preview = "The following duplicates will be removed:\n" + for data in duplicates.values(): + best = data.get("keep") + preview += f"\nKeeping: '{best.name}' (id: {best.id})" + + for duplicate in data.get("delete"): + preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" + preview += "\n" + + # Get confirmation and execute deletions + if TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message=preview, + prompt_title="Clean up duplicate suborganizations?", + verify_message="*** WARNING: This will delete suborganizations! ***" + ): + try: + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] + + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + ids_to_delete = [ + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ] + + # Bulk delete all duplicates + delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() + logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") + + except Exception as e: + logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 9cf4d36ea..b8a0ed091 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -104,7 +104,11 @@ class Command(BaseCommand): also create new suborganizations""" portfolio, created = self.create_portfolio(federal_agency) if created: - self.create_suborganizations(portfolio, federal_agency) + valid_agencies = DomainInformation.objects.filter( + federal_agency=federal_agency, organization_name__isnull=False + ) + org_names = set(valid_agencies.values_list("organization_name", flat=True)) + self.create_suborganizations(portfolio, federal_agency, org_names) if parse_domains or both: self.handle_portfolio_domains(portfolio, federal_agency) @@ -155,13 +159,8 @@ class Command(BaseCommand): return portfolio, True - def create_suborganizations(self, portfolio: Portfolio, federal_agency: FederalAgency): + def create_suborganizations(self, portfolio: Portfolio, federal_agency: FederalAgency, org_names: set): """Create Suborganizations tied to the given portfolio based on DomainInformation objects""" - valid_agencies = DomainInformation.objects.filter( - federal_agency=federal_agency, organization_name__isnull=False - ) - org_names = set(valid_agencies.values_list("organization_name", flat=True)) - if not org_names: message = ( "Could not add any suborganizations." @@ -232,6 +231,16 @@ class Command(BaseCommand): domain_request.portfolio = portfolio if domain_request.organization_name in suborgs: domain_request.sub_organization = suborgs.get(domain_request.organization_name) + else: + # Fill in the requesting suborg fields if we have the data to do so + if domain_request.organization_name and domain_request.city and domain_request.state_territory: + domain_request.requested_suborganization = domain_request.organization_name + domain_request.suborganization_city = domain_request.city + domain_request.suborganization_state_territory = domain_request.state_territory + else: + message = f"No suborganization data found whatsoever for {domain_request}." + TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) + self.updated_portfolios.add(portfolio) DomainRequest.objects.bulk_update(domain_requests, ["portfolio", "sub_organization"]) From f359a636d02103c0dd5421a5c25be4314fbc3a27 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 6 Jan 2025 13:45:37 -0500 Subject: [PATCH 017/201] combobox in domain request organization contact --- src/registrar/forms/domain_request_wizard.py | 17 ++++++++++++++--- src/registrar/forms/utility/combobox.py | 5 +++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 src/registrar/forms/utility/combobox.py diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 45bd575f9..f58d62c0e 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -7,6 +7,7 @@ from django import forms from django.core.validators import RegexValidator, MaxLengthValidator from django.utils.safestring import mark_safe +from registrar.forms.utility.combobox import ComboboxWidget from registrar.forms.utility.wizard_form_helper import ( RegistrarForm, RegistrarFormSet, @@ -257,6 +258,7 @@ class OrganizationContactForm(RegistrarForm): required=False, queryset=FederalAgency.objects.exclude(agency__in=excluded_agencies), empty_label="--Select--", + widget=ComboboxWidget, ) organization_name = forms.CharField( label="Organization name", @@ -298,11 +300,20 @@ class OrganizationContactForm(RegistrarForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Set initial value for federal agency combo box and specify combobox template - if self.domain_request and self.domain_request.federal_agency: + + # Initialize federal_agency combobox widget + # Domain requests forms have prefix associated with step + prefix = kwargs.get("prefix", "") + prefixed_name = f"{prefix}-federal_agency" if prefix else "federal_agency" + + # For combobox widget, need to set the data-default-value to selected value + if self.is_bound and self.data.get(prefixed_name): + # If form is bound (from a POST), use submitted value + self.fields["federal_agency"].widget.attrs["data-default-value"] = self.data.get(prefixed_name) + elif self.domain_request and self.domain_request.federal_agency: + # If form is not bound, set initial self.fields["federal_agency"].initial = self.domain_request.federal_agency self.fields["federal_agency"].widget.attrs["data-default-value"] = self.domain_request.federal_agency.pk - self.fields["federal_agency"].widget.template_name = "django/forms/widgets/combobox.html", def clean_federal_agency(self): """Require something to be selected when this is a federal agency.""" diff --git a/src/registrar/forms/utility/combobox.py b/src/registrar/forms/utility/combobox.py new file mode 100644 index 000000000..b7db16ccc --- /dev/null +++ b/src/registrar/forms/utility/combobox.py @@ -0,0 +1,5 @@ +from django.forms import Select + + +class ComboboxWidget(Select): + template_name = "django/forms/widgets/combobox.html" \ No newline at end of file From 4480e3255375120f3ca6612c0f15309ebd57aa9a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 6 Jan 2025 14:15:13 -0500 Subject: [PATCH 018/201] updated combobox widget to set proper data-default-value and set domain suborganization form to use new combobox widget --- src/registrar/forms/domain.py | 18 +++--------------- src/registrar/forms/domain_request_wizard.py | 17 ----------------- .../django/forms/widgets/combobox.html | 1 + 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index b43d91a58..1e4068125 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -4,6 +4,7 @@ import logging from django import forms from django.core.validators import MinValueValidator, MaxValueValidator, RegexValidator, MaxLengthValidator from django.forms import formset_factory +from registrar.forms.utility.combobox import ComboboxWidget from registrar.models import DomainRequest, FederalAgency from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.models.suborganization import Suborganization @@ -161,9 +162,10 @@ class DomainSuborganizationForm(forms.ModelForm): """Form for updating the suborganization""" sub_organization = forms.ModelChoiceField( + label = "Suborganization name", queryset=Suborganization.objects.none(), required=False, - widget=forms.Select(), + widget=ComboboxWidget, ) class Meta: @@ -178,20 +180,6 @@ class DomainSuborganizationForm(forms.ModelForm): portfolio = self.instance.portfolio if self.instance else None self.fields["sub_organization"].queryset = Suborganization.objects.filter(portfolio=portfolio) - # Set initial value - if self.instance and self.instance.sub_organization: - self.fields["sub_organization"].initial = self.instance.sub_organization - - # Set custom form label - self.fields["sub_organization"].label = "Suborganization name" - - # Use the combobox rather than the regular select widget - self.fields["sub_organization"].widget.template_name = "django/forms/widgets/combobox.html" - - # Set data-default-value attribute - if self.instance and self.instance.sub_organization: - self.fields["sub_organization"].widget.attrs["data-default-value"] = self.instance.sub_organization.pk - class BaseNameserverFormset(forms.BaseFormSet): def clean(self): diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index f58d62c0e..e090ac8d2 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -298,23 +298,6 @@ class OrganizationContactForm(RegistrarForm): label="Urbanization (required for Puerto Rico only)", ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Initialize federal_agency combobox widget - # Domain requests forms have prefix associated with step - prefix = kwargs.get("prefix", "") - prefixed_name = f"{prefix}-federal_agency" if prefix else "federal_agency" - - # For combobox widget, need to set the data-default-value to selected value - if self.is_bound and self.data.get(prefixed_name): - # If form is bound (from a POST), use submitted value - self.fields["federal_agency"].widget.attrs["data-default-value"] = self.data.get(prefixed_name) - elif self.domain_request and self.domain_request.federal_agency: - # If form is not bound, set initial - self.fields["federal_agency"].initial = self.domain_request.federal_agency - self.fields["federal_agency"].widget.attrs["data-default-value"] = self.domain_request.federal_agency.pk - def clean_federal_agency(self): """Require something to be selected when this is a federal agency.""" federal_agency = self.cleaned_data.get("federal_agency", None) diff --git a/src/registrar/templates/django/forms/widgets/combobox.html b/src/registrar/templates/django/forms/widgets/combobox.html index 7ff31945b..4fe796347 100644 --- a/src/registrar/templates/django/forms/widgets/combobox.html +++ b/src/registrar/templates/django/forms/widgets/combobox.html @@ -11,6 +11,7 @@ for now we just carry the attribute to both the parent element and the select. {{ name }}="{{ value }}" {% endif %} {% endfor %} +data-default-value="{% for group_name, group_choices, group_index in widget.optgroups %}{% for option in group_choices %}{% if option.selected %}{{ option.value }}{% endif %}{% endfor %}{% endfor %}" > {% include "django/forms/widgets/select.html" %}

      From 3146dc07fadd3d9b92e5932b7964f57943cb271a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 6 Jan 2025 14:22:36 -0500 Subject: [PATCH 019/201] applied widget to state territory --- src/registrar/forms/domain_request_wizard.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index e090ac8d2..fcba68de8 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -282,6 +282,7 @@ class OrganizationContactForm(RegistrarForm): error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, + widget=ComboboxWidget, ) zipcode = forms.CharField( label="Zip code", From 560baed2e1e2eb95aa48364b2e3673084814a741 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 6 Jan 2025 14:39:44 -0500 Subject: [PATCH 020/201] linted --- src/registrar/forms/domain.py | 2 +- src/registrar/forms/utility/combobox.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 1e4068125..87a52d142 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -162,7 +162,7 @@ class DomainSuborganizationForm(forms.ModelForm): """Form for updating the suborganization""" sub_organization = forms.ModelChoiceField( - label = "Suborganization name", + label="Suborganization name", queryset=Suborganization.objects.none(), required=False, widget=ComboboxWidget, diff --git a/src/registrar/forms/utility/combobox.py b/src/registrar/forms/utility/combobox.py index b7db16ccc..277aec4f3 100644 --- a/src/registrar/forms/utility/combobox.py +++ b/src/registrar/forms/utility/combobox.py @@ -2,4 +2,4 @@ from django.forms import Select class ComboboxWidget(Select): - template_name = "django/forms/widgets/combobox.html" \ No newline at end of file + template_name = "django/forms/widgets/combobox.html" From d0183d4d149c643d638c3fd9856ef90ef715ae67 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:42:49 -0700 Subject: [PATCH 021/201] cleanup --- .../commands/clean_duplicate_suborgs.py | 23 +++++-------------- .../commands/create_federal_portfolio.py | 2 +- .../models/utility/generic_helper.py | 6 +++++ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/registrar/management/commands/clean_duplicate_suborgs.py b/src/registrar/management/commands/clean_duplicate_suborgs.py index a5c7a87f0..2e44746ad 100644 --- a/src/registrar/management/commands/clean_duplicate_suborgs.py +++ b/src/registrar/management/commands/clean_duplicate_suborgs.py @@ -17,7 +17,7 @@ class Command(BaseCommand): for suborg in all_suborgs: # Normalize name by removing extra spaces and converting to lowercase - normalized_name = " ".join(suborg.name.split()).lower() + normalized_name = " ".join(suborg.name.trim().split()).lower() # First occurrence of this name if normalized_name not in duplicates: @@ -28,7 +28,8 @@ class Command(BaseCommand): continue # Compare with our current best - current_best = duplicates[normalized_name]["keep"] + duplicate_record = duplicates.get(normalized_name) + current_best = duplicate_record.get("keep") # Check if all other fields match. # If they don't, we should inspect this record manually. @@ -50,25 +51,13 @@ class Command(BaseCommand): # The fewest spaces and most capitals wins. new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) - # TODO - # Split into words and count properly capitalized first letters - # new_proper_caps = sum( - # 1 for word in suborg.name.split() - # if word and word[0].isupper() - # ) - # current_proper_caps = sum( - # 1 for word in current_best.name.split() - # if word and word[0].isupper() - # ) - # new_has_better_caps = new_proper_caps > current_proper_caps - if new_has_fewer_spaces or new_has_more_capitals: # New suborg is better - demote the old one to the delete list - duplicates[normalized_name]["delete"].append(current_best) - duplicates[normalized_name]["keep"] = suborg + duplicate_record["delete"].append(current_best) + duplicate_record["keep"] = suborg else: # If it is not better, just delete the old one - duplicates[normalized_name]["delete"].append(suborg) + duplicate_record["delete"].append(suborg) # Filter out entries without duplicates duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index b8a0ed091..15740cca9 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -107,7 +107,7 @@ class Command(BaseCommand): valid_agencies = DomainInformation.objects.filter( federal_agency=federal_agency, organization_name__isnull=False ) - org_names = set(valid_agencies.values_list("organization_name", flat=True)) + org_names = set([agency.trim() for agency in valid_agencies.values_list("organization_name", flat=True)]) self.create_suborganizations(portfolio, federal_agency, org_names) if parse_domains or both: self.handle_portfolio_domains(portfolio, federal_agency) diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 5e425f5a3..d1812d476 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -343,3 +343,9 @@ def value_of_attribute(obj, attribute_name: str): if callable(value): value = value() return value + + +def normalize_string(string_to_normalize: str) -> str: + """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" + new_string = " ".join(string_to_normalize.trim().split()) + return new_string.lower() From 6086b9587f10ab44aa0274a2289fa511663138ca Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:43:48 -0700 Subject: [PATCH 022/201] Remove unrelated content --- .../commands/clean_duplicate_suborgs.py | 112 ------------------ .../models/utility/generic_helper.py | 6 - 2 files changed, 118 deletions(-) delete mode 100644 src/registrar/management/commands/clean_duplicate_suborgs.py diff --git a/src/registrar/management/commands/clean_duplicate_suborgs.py b/src/registrar/management/commands/clean_duplicate_suborgs.py deleted file mode 100644 index 2e44746ad..000000000 --- a/src/registrar/management/commands/clean_duplicate_suborgs.py +++ /dev/null @@ -1,112 +0,0 @@ -import logging -from django.core.management import BaseCommand -from registrar.models import Suborganization, DomainRequest, DomainInformation -from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper - - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" - - def handle(self, **kwargs): - # Find duplicates - duplicates = {} - all_suborgs = Suborganization.objects.all() - - for suborg in all_suborgs: - # Normalize name by removing extra spaces and converting to lowercase - normalized_name = " ".join(suborg.name.trim().split()).lower() - - # First occurrence of this name - if normalized_name not in duplicates: - duplicates[normalized_name] = { - "keep": suborg, - "delete": [] - } - continue - - # Compare with our current best - duplicate_record = duplicates.get(normalized_name) - current_best = duplicate_record.get("keep") - - # Check if all other fields match. - # If they don't, we should inspect this record manually. - fields_to_compare = ["portfolio", "city", "state_territory"] - fields_match = all( - getattr(suborg, field) == getattr(current_best, field) - for field in fields_to_compare - ) - if not fields_match: - logger.warning( - f"{TerminalColors.YELLOW}" - f"\nSkipping potential duplicate: {suborg.name} (id: {suborg.id})" - f"\nData mismatch with {current_best.name} (id: {current_best.id})" - f"{TerminalColors.ENDC}" - ) - continue - - # Determine if new suborg is better than current best. - # The fewest spaces and most capitals wins. - new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") - new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) - if new_has_fewer_spaces or new_has_more_capitals: - # New suborg is better - demote the old one to the delete list - duplicate_record["delete"].append(current_best) - duplicate_record["keep"] = suborg - else: - # If it is not better, just delete the old one - duplicate_record["delete"].append(suborg) - - # Filter out entries without duplicates - duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} - if not duplicates: - logger.info(f"No duplicate suborganizations found.") - return - - # Show preview of changes - preview = "The following duplicates will be removed:\n" - for data in duplicates.values(): - best = data.get("keep") - preview += f"\nKeeping: '{best.name}' (id: {best.id})" - - for duplicate in data.get("delete"): - preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" - preview += "\n" - - # Get confirmation and execute deletions - if TerminalHelper.prompt_for_execution( - system_exit_on_terminate=True, - prompt_message=preview, - prompt_title="Clean up duplicate suborganizations?", - verify_message="*** WARNING: This will delete suborganizations! ***" - ): - try: - # Update all references to point to the right suborg before deletion - for record in duplicates.values(): - best_record = record.get("keep") - delete_ids = [dupe.id for dupe in record.get("delete")] - - # Update domain requests - DomainRequest.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - # Update domain information - DomainInformation.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - ids_to_delete = [ - dupe.id - for data in duplicates.values() - for dupe in data["delete"] - ] - - # Bulk delete all duplicates - delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() - logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") - - except Exception as e: - logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index d1812d476..5e425f5a3 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -343,9 +343,3 @@ def value_of_attribute(obj, attribute_name: str): if callable(value): value = value() return value - - -def normalize_string(string_to_normalize: str) -> str: - """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" - new_string = " ".join(string_to_normalize.trim().split()) - return new_string.lower() From 345ca0307e161fe76e982b941b18064edd704f43 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:46:36 -0700 Subject: [PATCH 023/201] Update create_federal_portfolio.py --- .../management/commands/create_federal_portfolio.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 15740cca9..f73f1ec66 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -104,11 +104,7 @@ class Command(BaseCommand): also create new suborganizations""" portfolio, created = self.create_portfolio(federal_agency) if created: - valid_agencies = DomainInformation.objects.filter( - federal_agency=federal_agency, organization_name__isnull=False - ) - org_names = set([agency.trim() for agency in valid_agencies.values_list("organization_name", flat=True)]) - self.create_suborganizations(portfolio, federal_agency, org_names) + self.create_suborganizations(portfolio, federal_agency) if parse_domains or both: self.handle_portfolio_domains(portfolio, federal_agency) @@ -159,8 +155,13 @@ class Command(BaseCommand): return portfolio, True - def create_suborganizations(self, portfolio: Portfolio, federal_agency: FederalAgency, org_names: set): + def create_suborganizations(self, portfolio: Portfolio, federal_agency: FederalAgency): """Create Suborganizations tied to the given portfolio based on DomainInformation objects""" + valid_agencies = DomainInformation.objects.filter( + federal_agency=federal_agency, organization_name__isnull=False + ) + org_names = set(valid_agencies.values_list("organization_name", flat=True)) + if not org_names: message = ( "Could not add any suborganizations." From 5e6a7987dfbd6ef9222c9adc66ead235f9433b8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:47:57 -0700 Subject: [PATCH 024/201] Update create_federal_portfolio.py --- .../management/commands/create_federal_portfolio.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index f73f1ec66..5fac24f53 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -233,14 +233,9 @@ class Command(BaseCommand): if domain_request.organization_name in suborgs: domain_request.sub_organization = suborgs.get(domain_request.organization_name) else: - # Fill in the requesting suborg fields if we have the data to do so - if domain_request.organization_name and domain_request.city and domain_request.state_territory: - domain_request.requested_suborganization = domain_request.organization_name - domain_request.suborganization_city = domain_request.city - domain_request.suborganization_state_territory = domain_request.state_territory - else: - message = f"No suborganization data found whatsoever for {domain_request}." - TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) + domain_request.requested_suborganization = domain_request.organization_name + domain_request.suborganization_city = domain_request.city + domain_request.suborganization_state_territory = domain_request.state_territory self.updated_portfolios.add(portfolio) From 9052d9ed3f3924293675879c9e6955ffb4cf06a4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:21:50 -0700 Subject: [PATCH 025/201] Use dedicated script --- .../commands/create_federal_portfolio.py | 5 --- .../commands/populate_requested_suborg.py | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 src/registrar/management/commands/populate_requested_suborg.py diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 5fac24f53..9cf4d36ea 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -232,11 +232,6 @@ class Command(BaseCommand): domain_request.portfolio = portfolio if domain_request.organization_name in suborgs: domain_request.sub_organization = suborgs.get(domain_request.organization_name) - else: - domain_request.requested_suborganization = domain_request.organization_name - domain_request.suborganization_city = domain_request.city - domain_request.suborganization_state_territory = domain_request.state_territory - self.updated_portfolios.add(portfolio) DomainRequest.objects.bulk_update(domain_requests, ["portfolio", "sub_organization"]) diff --git a/src/registrar/management/commands/populate_requested_suborg.py b/src/registrar/management/commands/populate_requested_suborg.py new file mode 100644 index 000000000..54f811413 --- /dev/null +++ b/src/registrar/management/commands/populate_requested_suborg.py @@ -0,0 +1,37 @@ +import logging +from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors, TerminalHelper +from registrar.models import DomainRequest + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + help = "Loops through each domain request object and populates the last_status_update and first_submitted_date" + + def handle(self, **kwargs): + """Loops through each DomainRequest object and populates + its last_status_update and first_submitted_date values""" + filter_conditions = {"portfolio__isnull": False, "sub_organization__isnull": True} + fields_to_update = ["requested_suborganization", "suborganization_city", "suborganization_state_territory"] + self.mass_update_records(DomainRequest, filter_conditions, fields_to_update) + + def update_record(self, record: DomainRequest): + """Adds data to requested_suborganization, suborganization_city, and suborganization_state_territory.""" + record.requested_suborganization = record.organization_name + record.suborganization_city = record.city + record.suborganization_state_territory = record.state_territory + message = ( + f"Updating {record} => requested_suborg: {record.organization_name}, " + f"sub_city: {record.city}, suborg_state_territory: {record.state_territory}." + ) + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKBLUE, message) + + def should_skip_record(self, record: DomainRequest) -> bool: + """Skips record update if we're missing org name, city, and state territory.""" + required_fields = [ + record.organization_name, + record.city, + record.state_territory + ] + return not all(bool(field) for field in required_fields) From 581dc7081d5a5946b51807c82841ee3df0b24938 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 6 Jan 2025 13:37:11 -0700 Subject: [PATCH 026/201] Make non-org Domain and Domain Request table columns align --- .../src/js/getgov/table-domain-requests.js | 24 ++++--------------- .../assets/src/js/getgov/table-domains.js | 6 ++++- .../assets/src/sass/_theme/_base.scss | 4 ++++ 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/table-domain-requests.js b/src/registrar/assets/src/js/getgov/table-domain-requests.js index 3078a0fbf..3d5d114c3 100644 --- a/src/registrar/assets/src/js/getgov/table-domain-requests.js +++ b/src/registrar/assets/src/js/getgov/table-domain-requests.js @@ -53,24 +53,6 @@ export class DomainRequestsTable extends BaseTable { this.toggleExportButton(data.domain_requests); let needsDeleteColumn = data.domain_requests.some(request => request.is_deletable); - - // Remove existing delete th and td if they exist - let existingDeleteTh = document.querySelector('.delete-header'); - if (!needsDeleteColumn) { - if (existingDeleteTh) - existingDeleteTh.remove(); - } else { - if (!existingDeleteTh) { - const delheader = document.createElement('th'); - delheader.setAttribute('scope', 'col'); - delheader.setAttribute('role', 'columnheader'); - delheader.setAttribute('class', 'delete-header width-5'); - delheader.innerHTML = ` - Delete Action`; - let tableHeaderRow = this.tableWrapper.querySelector('thead tr'); - tableHeaderRow.appendChild(delheader); - } - } return { 'needsAdditionalColumn': needsDeleteColumn }; } @@ -88,8 +70,12 @@ export class DomainRequestsTable extends BaseTable { Domain request cannot be deleted now. Edit the request for more information.`; let markupCreatorRow = ''; + + // Forces columns of the domain request and domain tables to align in non-org views + let columnWidthLimiterClass = 'width-quarter'; if (this.portfolioValue) { + columnWidthLimiterClass = ''; markupCreatorRow = ` ${request.creator ? request.creator : ''} @@ -133,7 +119,7 @@ export class DomainRequestsTable extends BaseTable { ${request.status} - +
      ${suborganization} @@ -55,7 +59,7 @@ export class DomainsTable extends BaseTable { ${markupForSuborganizationRow} - +
      +
      + {% for error in form.is_policy_acknowledged.errors %} +

      {{ error }}

      + {% endfor %} +
      +
      + {% endif %} + {% if portfolio %}
      +
      {% url 'user-profile' as url %} {% include "includes/summary_item.html" with title='Your Contact Information' value=user edit_link=url editable=is_editable contact='true' %} @@ -66,62 +78,50 @@ {% include "includes/summary_item.html" with title='Domain managers' list=True users=True value=domain.permissions.all edit_link=url editable=is_editable %} {% endif %} - +
      -

      - Acknowledgement of .gov domain requirements -

      + Acknowledgement of .gov domain requirements       - - {% if messages %} -
        - {% for message in messages %} -

        {{ message }}

        - {% endfor %} - - {% endif %} - - -
        - {% csrf_token %} -
        -
        - - - - + . + * + +
        + + + + +
        {% endblock %} {# domain_content #} \ No newline at end of file diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index ee8adf903..0a4096fbf 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -454,7 +454,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): self.user.save() - def todays_expiration_date(self): + def expiration_date_one_year_out(self): todays_date = datetime.today() new_expiration_date = todays_date.replace(year=todays_date.year + 1) return new_expiration_date @@ -466,7 +466,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): return True def custom_renew_domain(self): - self.domain_with_ip.expiration_date = self.todays_expiration_date() + self.domain_with_ip.expiration_date = self.expiration_date_one_year_out() self.domain_with_ip.save() @override_flag("domain_renewal", active=True) @@ -659,7 +659,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): self.assertRedirects(response, reverse("domain", kwargs={"pk": self.domain_with_ip.id})) # Check for the updated expiration - formatted_new_expiration_date = self.todays_expiration_date().strftime("%b. %-d, %Y") + formatted_new_expiration_date = self.expiration_date_one_year_out().strftime("%b. %-d, %Y") redirect_response = self.client.get(reverse("domain", kwargs={"pk": self.domain_with_ip.id}), follow=True) self.assertContains(redirect_response, formatted_new_expiration_date) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b849459f2..593fc9093 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -12,11 +12,11 @@ from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin from django.db import IntegrityError from django.http import HttpResponseRedirect -from django.shortcuts import redirect, render +from django.shortcuts import redirect, render, get_object_or_404 from django.urls import reverse from django.views.generic.edit import FormMixin from django.conf import settings -from registrar.forms.domain import DomainSuborganizationForm +from registrar.forms.domain import DomainSuborganizationForm, DomainRenewalForm from registrar.models import ( Domain, DomainRequest, @@ -364,30 +364,32 @@ class DomainRenewalView(DomainBaseView): self._update_session_with_domain() def post(self, request, pk): - domain = Domain.objects.filter(id=pk).first() + domain = get_object_or_404(Domain, id=pk) - # Check if the checkbox is checked - is_policy_acknowledged = request.POST.get("is_policy_acknowledged", None) - if is_policy_acknowledged != "on": - messages.error( - request, "Check the box if you read and agree to the requirements for operating a .gov domain." - ) - return render( - request, - "domain_renewal.html", - { - "domain": domain, - "form": request.POST, - }, - ) + form = DomainRenewalForm(request.POST) - if "submit_button" in request.POST: - try: - domain.renew_domain() - messages.success(request, "This domain has been renewed for one year.") - except Exception as e: - messages.error(request, "This domain has not been renewed for one year, error was %s" % e) - return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) + if form.is_valid(): + # check for key in the post request data + if "submit_button" in request.POST: + try: + domain.renew_domain() + messages.success(request, "This domain has been renewed for one year.") + except Exception as e: + messages.error( + request, + "This domain has not been renewed for one year, please email help@get.gov if this problem persists.", + ) + return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) + + # if not valid, render the template with error messages + return render( + request, + "domain_renewal.html", + { + "domain": domain, + "form": form, + }, + ) class DomainOrgNameAddressView(DomainFormBaseView): From e196e02d9284c55057ec1dc6ce4736c6b0f3e0ba Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 7 Jan 2025 17:28:35 -0800 Subject: [PATCH 053/201] Address linter errors --- src/registrar/views/domain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 593fc9093..4b26caa5f 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -374,10 +374,11 @@ class DomainRenewalView(DomainBaseView): try: domain.renew_domain() messages.success(request, "This domain has been renewed for one year.") - except Exception as e: + except Exception: messages.error( request, - "This domain has not been renewed for one year, please email help@get.gov if this problem persists.", + "This domain has not been renewed for one year, " + "please email help@get.gov if this problem persists.", ) return HttpResponseRedirect(reverse("domain", kwargs={"pk": pk})) From aad1ee976b5735633d93caa1a1b9724a46206206 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 7 Jan 2025 17:40:37 -0800 Subject: [PATCH 054/201] Fix checkbox unchecked test --- src/registrar/tests/test_views_domain.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 0a4096fbf..ef4d415df 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -631,19 +631,11 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): # Grab the renewal URL renewal_url = reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id}) - # Test clicking the checkbox + # Test that the checkbox is not checked response = self.client.post(renewal_url, data={"submit_button": "next"}) - # Verify the error message is displayed - # Retrieves messages obj (used in the post call) - messages = list(get_messages(response.wsgi_request)) - # Check we only get 1 error message - self.assertEqual(len(messages), 1) - # Check that the 1 error msg also is the right text - self.assertEqual( - str(messages[0]), - "Check the box if you read and agree to the requirements for operating a .gov domain.", - ) + error_message = "Check the box if you read and agree to the requirements for operating a .gov domain." + self.assertContains(response, error_message) @override_flag("domain_renewal", active=True) def test_ack_checkbox_checked(self): From 33e36590a4e0ff9b21836d22848618bc456b0533 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Tue, 7 Jan 2025 17:43:54 -0800 Subject: [PATCH 055/201] Remove unused import for linter --- src/registrar/tests/test_views_domain.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index ef4d415df..79240286a 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -11,7 +11,6 @@ from api.tests.common import less_console_noise_decorator from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from .common import MockEppLib, MockSESClient, create_user # type: ignore from django_webtest import WebTest # type: ignore -from django.contrib.messages import get_messages import boto3_mocking # type: ignore from registrar.utility.errors import ( From 018967e0ac8186770ace3a1a2c99956165d69cc1 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 7 Jan 2025 21:37:46 -0700 Subject: [PATCH 056/201] PR feedback --- src/registrar/assets/src/js/getgov/table-base.js | 4 ++-- .../assets/src/js/getgov/table-domain-requests.js | 13 +++++-------- src/registrar/assets/src/js/getgov/table-domains.js | 6 +----- src/registrar/assets/src/js/getgov/table-members.js | 8 ++++---- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/table-base.js b/src/registrar/assets/src/js/getgov/table-base.js index e1d5c11ce..4b7133d64 100644 --- a/src/registrar/assets/src/js/getgov/table-base.js +++ b/src/registrar/assets/src/js/getgov/table-base.js @@ -382,7 +382,7 @@ export class BaseTable { * for a member, they will also see the kebab column) * @param {Object} dataObjects - Data which contains info on domain requests or a user's permission * Currently returns a dictionary of either: - * - "needsAdditionalColumn": If a new column should be displayed + * - "isDeletable": If a new column should be displayed * - "UserPortfolioPermissionChoices": A user's portfolio permission choices */ customizeTable(dataObjects){ return {}; } @@ -406,7 +406,7 @@ export class BaseTable { * Returns either: data.members, data.domains or data.domain_requests * @param {Object} dataObject - The data used to populate the row content * @param {HTMLElement} tbody - The table body to which the new row is appended to - * @param {Object} customTableOptions - Additional options for customizing row appearance (ie needsAdditionalColumn) + * @param {Object} customTableOptions - Additional options for customizing row appearance (ie isDeletable) */ addRow(dataObject, tbody, customTableOptions) { throw new Error('addRow must be defined'); diff --git a/src/registrar/assets/src/js/getgov/table-domain-requests.js b/src/registrar/assets/src/js/getgov/table-domain-requests.js index 3d5d114c3..722ad1eef 100644 --- a/src/registrar/assets/src/js/getgov/table-domain-requests.js +++ b/src/registrar/assets/src/js/getgov/table-domain-requests.js @@ -53,7 +53,7 @@ export class DomainRequestsTable extends BaseTable { this.toggleExportButton(data.domain_requests); let needsDeleteColumn = data.domain_requests.some(request => request.is_deletable); - return { 'needsAdditionalColumn': needsDeleteColumn }; + return { 'isDeletable': needsDeleteColumn }; } addRow(dataObject, tbody, customTableOptions) { @@ -71,11 +71,8 @@ export class DomainRequestsTable extends BaseTable { let markupCreatorRow = ''; - // Forces columns of the domain request and domain tables to align in non-org views - let columnWidthLimiterClass = 'width-quarter'; if (this.portfolioValue) { - columnWidthLimiterClass = ''; markupCreatorRow = ` ${request.creator ? request.creator : ''} @@ -119,15 +116,15 @@ export class DomainRequestsTable extends BaseTable { ${request.status} - -
        - + +
        + ${actionLabel} ${request.requested_domain ? request.requested_domain : 'New domain request'} - ${customTableOptions.needsAdditionalColumn ? modalTrigger : ''} + ${customTableOptions.isDeletable ? modalTrigger : ''}
        `; diff --git a/src/registrar/assets/src/js/getgov/table-domains.js b/src/registrar/assets/src/js/getgov/table-domains.js index deeb04988..51abf0c6a 100644 --- a/src/registrar/assets/src/js/getgov/table-domains.js +++ b/src/registrar/assets/src/js/getgov/table-domains.js @@ -23,12 +23,8 @@ export class DomainsTable extends BaseTable { const row = document.createElement('tr'); let markupForSuborganizationRow = ''; - - // Forces columns of the domain request and domain tables to align in non-org views - let columnWidthLimiterClass = 'width-quarter'; if (this.portfolioValue) { - columnWidthLimiterClass = '' markupForSuborganizationRow = ` ${suborganization} @@ -59,7 +55,7 @@ export class DomainsTable extends BaseTable { ${markupForSuborganizationRow} - + ${member.name} - ${customTableOptions.needsAdditionalColumn ? ''+kebabHTML+'' : ''} + ${customTableOptions.isDeletable ? ''+kebabHTML+'' : ''} `; tbody.appendChild(row); if (domainsHTML || permissionsHTML) { @@ -137,7 +137,7 @@ export class MembersTable extends BaseTable { } // This easter egg is only for fixtures that dont have names as we are displaying their emails // All prod users will have emails linked to their account - if (customTableOptions.needsAdditionalColumn) MembersTable.addMemberModal(num_domains, member.email || "Samwise Gamgee", member_delete_url, unique_id, row); + if (customTableOptions.isDeletable) MembersTable.addMemberModal(num_domains, member.email || "Samwise Gamgee", member_delete_url, unique_id, row); } /** From 5ca74fdc5374b864db3aa6e32b6d2c5ebed3882c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:38:50 -0700 Subject: [PATCH 057/201] Finish unit tests --- .../commands/create_federal_portfolio.py | 32 ++++++++---- .../tests/test_management_scripts.py | 50 +++++++++++++++++-- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 8ff32824b..ddcb5a4a9 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -79,7 +79,7 @@ class Command(BaseCommand): else: raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") - portfolio_set = set() + portfolios = [] for federal_agency in agencies: message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) @@ -87,7 +87,7 @@ class Command(BaseCommand): # C901 'Command.handle' is too complex (12) # We currently only grab the list of changed domain requests, but we may want to grab the domains too portfolio = self.handle_populate_portfolio(federal_agency, parse_domains, parse_requests, both) - portfolio_set.add(portfolio) + portfolios.append(portfolio) except Exception as exec: self.failed_portfolios.add(federal_agency) logger.error(exec) @@ -106,9 +106,14 @@ class Command(BaseCommand): # POST PROCESSING STEP: Remove the federal agency if it matches the portfolio name. # We only do this for started domain requests. if parse_requests: - self.post_process_started_domain_requests(portfolio_set) + TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message="This action will update domain requests even if they aren't on a portfolio.", + prompt_title="Do you want to clear federal agency on started domain requests?", + ) + self.post_process_started_domain_requests(agencies, portfolios) - def post_process_started_domain_requests(self, portfolio_set): + def post_process_started_domain_requests(self, agencies, portfolios): """ Removes duplicate organization data by clearing federal_agency when it matches the portfolio name. Only processes domain requests in STARTED status. @@ -116,15 +121,24 @@ class Command(BaseCommand): message = "Removing duplicate portfolio and federal_agency values from domain requests..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + # For each request, clear the federal agency under these conditions: + # 1. A portfolio *already exists* with the same name as the federal agency. + # 2. Said portfolio (or portfolios) are only the ones specified at the start of the script. + # 3. The domain request is in status "started". + # Note: Both names are normalized so excess spaces are stripped and the string is lowercased. domain_requests_to_update = DomainRequest.objects.filter( - portfolio__in=portfolio_set, - status=DomainRequest.DomainRequestStatus.STARTED, + federal_agency__in=agencies, federal_agency__agency__isnull=False, - portfolio__organization_name__isnull=False, + status=DomainRequest.DomainRequestStatus.STARTED, + organization_name__isnull=False, ) + portfolio_set = {normalize_string(portfolio.organization_name) for portfolio in portfolios if portfolio} + + # Update the request, assuming the given agency name matches the portfolio name updated_requests = [] for req in domain_requests_to_update: - if normalize_string(req.federal_agency.agency) == normalize_string(req.portfolio.organization_name): + agency_name = normalize_string(req.federal_agency.agency) + if agency_name in portfolio_set: req.federal_agency = None updated_requests.append(req) DomainRequest.objects.bulk_update(updated_requests, ["federal_agency"]) @@ -140,8 +154,6 @@ class Command(BaseCommand): def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations. - - Returns the processed portfolio """ portfolio, created = self.create_portfolio(federal_agency) if created: diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 882396f1e..8ecb7cbea 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1424,7 +1424,6 @@ class TestCreateFederalPortfolio(TestCase): # Create an agency wih no federal type (can only be created via specifiying it manually) self.federal_agency = FederalAgency.objects.create(agency="Test Federal Agency") - self.federal_agency_2 = FederalAgency.objects.create(agency="Sugarcane") # And create some with federal_type ones with creative names self.executive_agency_1 = FederalAgency.objects.create( @@ -1518,8 +1517,13 @@ class TestCreateFederalPortfolio(TestCase): call_command("create_federal_portfolio", **kwargs) @less_console_noise_decorator - def test_handle_portfolio_requests_sync_federal_agency(self): - """Test that federal agency is cleared when org name matches portfolio name""" + def test_post_process_started_domain_requests_existing_portfolio(self): + """Ensures that federal agency is cleared when agency name matches portfolio name. + As the name implies, this implicitly tests the "post_process_started_domain_requests" function. + """ + federal_agency_2 = FederalAgency.objects.create(agency="Sugarcane", federal_type=BranchChoices.EXECUTIVE) + + # Test records with portfolios and no org names # Create a portfolio. This script skips over "started" portfolio = Portfolio.objects.create(organization_name="Sugarcane", creator=self.user) # Create a domain request with matching org name @@ -1527,7 +1531,7 @@ class TestCreateFederalPortfolio(TestCase): name="matching.gov", status=DomainRequest.DomainRequestStatus.STARTED, generic_org_type=DomainRequest.OrganizationChoices.FEDERAL, - federal_agency=self.federal_agency_2, + federal_agency=federal_agency_2, user=self.user, portfolio=portfolio, ) @@ -1558,6 +1562,44 @@ class TestCreateFederalPortfolio(TestCase): self.assertIsNotNone(matching_request_in_wrong_status.portfolio) self.assertEqual(matching_request_in_wrong_status.portfolio.organization_name, "Test Federal Agency") + @less_console_noise_decorator + def test_post_process_started_domain_requests(self): + """Tests that federal agency is cleared when agency name + matches an existing portfolio's name, even if the domain request isn't + directly on that portfolio.""" + + federal_agency_2 = FederalAgency.objects.create(agency="Sugarcane", federal_type=BranchChoices.EXECUTIVE) + + # Create a request with matching federal_agency name but no direct portfolio association + matching_agency_request = completed_domain_request( + name="agency-match.gov", + status=DomainRequest.DomainRequestStatus.STARTED, + generic_org_type=DomainRequest.OrganizationChoices.FEDERAL, + federal_agency=federal_agency_2, + user=self.user, + ) + + # Create a control request that shouldn't match + non_matching_request = completed_domain_request( + name="no-match.gov", + status=DomainRequest.DomainRequestStatus.STARTED, + generic_org_type=DomainRequest.OrganizationChoices.FEDERAL, + federal_agency=self.federal_agency, + user=self.user, + ) + + # We expect the matching agency to have its fed agency cleared. + self.run_create_federal_portfolio(agency_name="Sugarcane", parse_requests=True) + matching_agency_request.refresh_from_db() + non_matching_request.refresh_from_db() + + # Request with matching agency name should have federal_agency cleared + self.assertIsNone(matching_agency_request.federal_agency) + + # Non-matching request should keep its federal_agency + self.assertIsNotNone(non_matching_request.federal_agency) + self.assertEqual(non_matching_request.federal_agency, self.federal_agency) + @less_console_noise_decorator def test_create_single_portfolio(self): """Test portfolio creation with suborg and senior official.""" From bc92270897cfc8967894e874f3cdddb39c8e78c6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:49:04 -0700 Subject: [PATCH 058/201] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index ddcb5a4a9..72548aa08 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -85,7 +85,6 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) try: # C901 'Command.handle' is too complex (12) - # We currently only grab the list of changed domain requests, but we may want to grab the domains too portfolio = self.handle_populate_portfolio(federal_agency, parse_domains, parse_requests, both) portfolios.append(portfolio) except Exception as exec: @@ -105,11 +104,12 @@ class Command(BaseCommand): # POST PROCESSING STEP: Remove the federal agency if it matches the portfolio name. # We only do this for started domain requests. - if parse_requests: + if parse_requests or both: TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, prompt_message="This action will update domain requests even if they aren't on a portfolio.", - prompt_title="Do you want to clear federal agency on started domain requests?", + prompt_title="Do you want to clear federal agency on (related) started domain requests?", + verify_message=None ) self.post_process_started_domain_requests(agencies, portfolios) @@ -141,15 +141,18 @@ class Command(BaseCommand): if agency_name in portfolio_set: req.federal_agency = None updated_requests.append(req) - DomainRequest.objects.bulk_update(updated_requests, ["federal_agency"]) - # Log the results + # Execute the update and Log the results if TerminalHelper.prompt_for_execution( system_exit_on_terminate=False, - prompt_message=f"Updated {len(updated_requests)} domain requests successfully.", - prompt_title="Do you want to see a list of all changed domain requests?", + prompt_message=( + f"{len(domain_requests_to_update)} domain requests will be updated. " + f"These records will be changed: {[str(req) for req in updated_requests]}" + ), + prompt_title="Do wish to commit this update to the database?", ): - logger.info(f"Federal agency set to none on: {[str(request) for request in updated_requests]}") + DomainRequest.objects.bulk_update(updated_requests, ["federal_agency"]) + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKBLUE, "Action completed successfully.") def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will From b9dc92808440a5b124e6d146193302156920ee8c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:56:06 -0700 Subject: [PATCH 059/201] lint --- src/registrar/management/commands/create_federal_portfolio.py | 4 ++-- src/registrar/models/utility/generic_helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 72548aa08..d0fb14a9f 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -109,7 +109,7 @@ class Command(BaseCommand): system_exit_on_terminate=True, prompt_message="This action will update domain requests even if they aren't on a portfolio.", prompt_title="Do you want to clear federal agency on (related) started domain requests?", - verify_message=None + verify_message=None, ) self.post_process_started_domain_requests(agencies, portfolios) @@ -248,7 +248,7 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) else: new_suborgs.append( - Suborganization(name=normalize_string(name, lowercase=False), portfolio=portfolio) + Suborganization(name=name, portfolio=portfolio) ) # type: ignore if new_suborgs: diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 3a8da508e..e8992acc2 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -345,7 +345,7 @@ def value_of_attribute(obj, attribute_name: str): return value -def normalize_string(string_to_normalize: str, lowercase=True) -> str: +def normalize_string(string_to_normalize, lowercase=True): """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" if not isinstance(string_to_normalize, str): logger.error(f"normalize_string => {string_to_normalize} is not type str.") From b27f45b8d782f1042f432f82a4ce618bdb219fcd Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 8 Jan 2025 11:00:23 -0700 Subject: [PATCH 060/201] resolved merge conflict --- src/registrar/assets/src/js/getgov/table-members.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/src/js/getgov/table-members.js b/src/registrar/assets/src/js/getgov/table-members.js index be149a059..6b1a485cb 100644 --- a/src/registrar/assets/src/js/getgov/table-members.js +++ b/src/registrar/assets/src/js/getgov/table-members.js @@ -137,7 +137,7 @@ export class MembersTable extends BaseTable { } // This easter egg is only for fixtures that dont have names as we are displaying their emails // All prod users will have emails linked to their account - if (customTableOptions.needsAdditionalColumn) MembersTable.addMemberDeleteModal(num_domains, member.email || "Samwise Gamgee", member_delete_url, unique_id, row); + if (customTableOptions.isDeletable) MembersTable.addMemberDeleteModal(num_domains, member.email || "Samwise Gamgee", member_delete_url, unique_id, row); } /** From 628c4c0d0e4015a248bcc2ee33d583bbf5bcdb69 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Wed, 8 Jan 2025 13:56:48 -0500 Subject: [PATCH 061/201] expired domain renewal form --- src/registrar/templates/domain_detail.html | 6 ++++++ src/registrar/templates/domain_sidebar.html | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index b168f7e82..becf46d5b 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -50,11 +50,17 @@ {% if domain.get_state_help_text %}
        {% if has_domain_renewal_flag and domain.is_expiring and is_domain_manager %} + This domain has expired, but it is still online. + {% url 'domain-renewal' pk=domain.id as url %} + Renew to maintain access. + {% elif has_domain_renewal_flag and domain.is_expired and is_domain_manager %} This domain will expire soon. {% url 'domain-renewal' pk=domain.id as url %} Renew to maintain access. {% elif has_domain_renewal_flag and domain.is_expiring and is_portfolio_user %} This domain will expire soon. Contact one of the listed domain managers to renew the domain. + {% elif has_domain_renewal_flag and domain.is_expired and is_portfolio_user %} + This domain has expired, but it is still online. Contact one of the listed domain managers to renew the domain. {% else %} {{ domain.get_state_help_text }} {% endif %} diff --git a/src/registrar/templates/domain_sidebar.html b/src/registrar/templates/domain_sidebar.html index 9d71ebf63..c9feb7200 100644 --- a/src/registrar/templates/domain_sidebar.html +++ b/src/registrar/templates/domain_sidebar.html @@ -80,7 +80,7 @@ {% include "includes/domain_sidenav_item.html" with item_text="Domain managers" %} {% endwith %} - {% if has_domain_renewal_flag and is_domain_manager and domain.is_expiring %} + {% if has_domain_renewal_flag and is_domain_manager and (domain.is_expiring or domain.is_expired) %} {% with url_name="domain-renewal" %} {% include "includes/domain_sidenav_item.html" with item_text="Renewal form" %} {% endwith %} From 6da172c0285ad0c568216311ecbe397b9103a81f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 12:25:17 -0700 Subject: [PATCH 062/201] Update create_federal_portfolio.py --- .../management/commands/create_federal_portfolio.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index d0fb14a9f..4f4002da4 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -108,7 +108,9 @@ class Command(BaseCommand): TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, prompt_message="This action will update domain requests even if they aren't on a portfolio.", - prompt_title="Do you want to clear federal agency on (related) started domain requests?", + prompt_title=( + "POST PROCESS STEP: Do you want to clear federal agency on (related) started domain requests?" + ), verify_message=None, ) self.post_process_started_domain_requests(agencies, portfolios) @@ -247,9 +249,7 @@ class Command(BaseCommand): ) TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) else: - new_suborgs.append( - Suborganization(name=name, portfolio=portfolio) - ) # type: ignore + new_suborgs.append(Suborganization(name=name, portfolio=portfolio)) # type: ignore if new_suborgs: Suborganization.objects.bulk_create(new_suborgs) From e8fede74ac05885d3b83d90b5cff372f870a5c89 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Wed, 8 Jan 2025 15:25:06 -0500 Subject: [PATCH 063/201] updated code to fix parsing error --- src/registrar/templates/domain_sidebar.html | 10 +-- src/registrar/tests/test_views_domain.py | 77 +++++++++++++++------ 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/src/registrar/templates/domain_sidebar.html b/src/registrar/templates/domain_sidebar.html index c9feb7200..336a9fe7c 100644 --- a/src/registrar/templates/domain_sidebar.html +++ b/src/registrar/templates/domain_sidebar.html @@ -80,10 +80,12 @@ {% include "includes/domain_sidenav_item.html" with item_text="Domain managers" %} {% endwith %} - {% if has_domain_renewal_flag and is_domain_manager and (domain.is_expiring or domain.is_expired) %} - {% with url_name="domain-renewal" %} - {% include "includes/domain_sidenav_item.html" with item_text="Renewal form" %} - {% endwith %} + {% if has_domain_renewal_flag and is_domain_manager%} + {% if domain.is_expiring or domain.is_expired %} + {% with url_name="domain-renewal" %} + {% include "includes/domain_sidenav_item.html" with item_text="Renewal form" %} + {% endwith %} + {% endif %} {% endif %} {% endif %} diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index ee8adf903..406e96ed4 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -440,15 +440,15 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): username="usertest", ) - self.expiringdomain, _ = Domain.objects.get_or_create( - name="expiringdomain.gov", + self.domaintorenew, _ = Domain.objects.get_or_create( + name="domainrenewal.gov", ) UserDomainRole.objects.get_or_create( - user=self.user, domain=self.expiringdomain, role=UserDomainRole.Roles.MANAGER + user=self.user, domain=self.domaintorenew, role=UserDomainRole.Roles.MANAGER ) - DomainInformation.objects.get_or_create(creator=self.user, domain=self.expiringdomain) + DomainInformation.objects.get_or_create(creator=self.user, domain=self.domaintorenew) self.portfolio, _ = Portfolio.objects.get_or_create(organization_name="Test org", creator=self.user) @@ -459,9 +459,12 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): new_expiration_date = todays_date.replace(year=todays_date.year + 1) return new_expiration_date - def custom_is_expired(self): + def custom_is_expired_false(self): return False + def custom_is_expired_true(self): + return True + def custom_is_expiring(self): return True @@ -473,11 +476,11 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): def test_expiring_domain_on_detail_page_as_domain_manager(self): self.client.force_login(self.user) with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( - Domain, "is_expired", self.custom_is_expired + Domain, "is_expired", self.custom_is_expired_false ): - self.assertEquals(self.expiringdomain.state, Domain.State.UNKNOWN) + self.assertEquals(self.domaintorenew.state, Domain.State.UNKNOWN) detail_page = self.client.get( - reverse("domain", kwargs={"pk": self.expiringdomain.id}), + reverse("domain", kwargs={"pk": self.domaintorenew.id}), ) self.assertContains(detail_page, "Expiring soon") @@ -508,17 +511,17 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): UserPortfolioPermissionChoices.VIEW_ALL_DOMAINS, ], ) - expiringdomain2, _ = Domain.objects.get_or_create(name="bogusdomain2.gov") + domaintorenew2, _ = Domain.objects.get_or_create(name="bogusdomain2.gov") DomainInformation.objects.get_or_create( - creator=non_dom_manage_user, domain=expiringdomain2, portfolio=self.portfolio + creator=non_dom_manage_user, domain=domaintorenew2, portfolio=self.portfolio ) non_dom_manage_user.refresh_from_db() self.client.force_login(non_dom_manage_user) with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( - Domain, "is_expired", self.custom_is_expired + Domain, "is_expired", self.custom_is_expired_false ): detail_page = self.client.get( - reverse("domain", kwargs={"pk": expiringdomain2.id}), + reverse("domain", kwargs={"pk": domaintorenew2.id}), ) self.assertContains(detail_page, "Contact one of the listed domain managers to renew the domain.") @@ -527,29 +530,29 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): def test_expiring_domain_on_detail_page_in_org_model_as_a_domain_manager(self): portfolio, _ = Portfolio.objects.get_or_create(organization_name="Test org2", creator=self.user) - expiringdomain3, _ = Domain.objects.get_or_create(name="bogusdomain3.gov") + domaintorenew3, _ = Domain.objects.get_or_create(name="bogusdomain3.gov") - UserDomainRole.objects.get_or_create(user=self.user, domain=expiringdomain3, role=UserDomainRole.Roles.MANAGER) - DomainInformation.objects.get_or_create(creator=self.user, domain=expiringdomain3, portfolio=portfolio) + UserDomainRole.objects.get_or_create(user=self.user, domain=domaintorenew3, role=UserDomainRole.Roles.MANAGER) + DomainInformation.objects.get_or_create(creator=self.user, domain=domaintorenew3, portfolio=portfolio) self.user.refresh_from_db() self.client.force_login(self.user) with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( - Domain, "is_expired", self.custom_is_expired + Domain, "is_expired", self.custom_is_expired_false ): detail_page = self.client.get( - reverse("domain", kwargs={"pk": expiringdomain3.id}), + reverse("domain", kwargs={"pk": domaintorenew3.id}), ) self.assertContains(detail_page, "Renew to maintain access") @override_flag("domain_renewal", active=True) - def test_domain_renewal_form_and_sidebar(self): + def test_domain_renewal_form_and_sidebar_expiring(self): self.client.force_login(self.user) with patch.object(Domain, "is_expiring", self.custom_is_expiring), patch.object( - Domain, "is_expired", self.custom_is_expired + Domain, "is_expiring", self.custom_is_expiring ): # Grab the detail page detail_page = self.client.get( - reverse("domain", kwargs={"pk": self.expiringdomain.id}), + reverse("domain", kwargs={"pk": self.domaintorenew.id}), ) # Make sure we see the link as a domain manager @@ -559,14 +562,44 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): self.assertContains(detail_page, "Renewal form") # Grab link to the renewal page - renewal_form_url = reverse("domain-renewal", kwargs={"pk": self.expiringdomain.id}) + renewal_form_url = reverse("domain-renewal", kwargs={"pk": self.domaintorenew.id}) self.assertContains(detail_page, f'href="{renewal_form_url}"') # Simulate clicking the link response = self.client.get(renewal_form_url) self.assertEqual(response.status_code, 200) - self.assertContains(response, f"Renew {self.expiringdomain.name}") + self.assertContains(response, f"Renew {self.domaintorenew.name}") + + @override_flag("domain_renewal", active=True) + def test_domain_renewal_form_and_sidebar_expired(self): + + self.client.force_login(self.user) + + with patch.object(Domain, "is_expired", self.custom_is_expired_true), patch.object( + Domain, "is_expired", self.custom_is_expired_true + ): + # Grab the detail page + detail_page = self.client.get( + reverse("domain", kwargs={"pk": self.domaintorenew.id}), + ) + + print("puglesss", self.domaintorenew.is_expired) + # Make sure we see the link as a domain manager + self.assertContains(detail_page, "Renew to maintain access") + + # Make sure we can see Renewal form on the sidebar since it's expired + self.assertContains(detail_page, "Renewal form") + + # Grab link to the renewal page + renewal_form_url = reverse("domain-renewal", kwargs={"pk": self.domaintorenew.id}) + self.assertContains(detail_page, f'href="{renewal_form_url}"') + + # Simulate clicking the link + response = self.client.get(renewal_form_url) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, f"Renew {self.domaintorenew.name}") @override_flag("domain_renewal", active=True) def test_domain_renewal_form_your_contact_info_edit(self): From c7af6645d4a8ffac6a3b33b307f97ed6871aded2 Mon Sep 17 00:00:00 2001 From: asaki222 Date: Wed, 8 Jan 2025 15:31:59 -0500 Subject: [PATCH 064/201] ran app black . --- src/registrar/tests/test_views_domain.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 406e96ed4..575dc0faf 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -464,7 +464,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): def custom_is_expired_true(self): return True - + def custom_is_expiring(self): return True @@ -570,12 +570,12 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): self.assertEqual(response.status_code, 200) self.assertContains(response, f"Renew {self.domaintorenew.name}") - + @override_flag("domain_renewal", active=True) def test_domain_renewal_form_and_sidebar_expired(self): - + self.client.force_login(self.user) - + with patch.object(Domain, "is_expired", self.custom_is_expired_true), patch.object( Domain, "is_expired", self.custom_is_expired_true ): From 5e5626aaba19173aa9a7cfb7b9f43031bcb8e486 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:54:54 -0700 Subject: [PATCH 065/201] basic logic --- .../commands/create_federal_portfolio.py | 94 +++++++++++-- .../commands/patch_suborganizations.py | 129 ++++++++++++++++++ 2 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 src/registrar/management/commands/patch_suborganizations.py diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 9cf4d36ea..f3debba56 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -5,7 +5,7 @@ import logging from django.core.management import BaseCommand, CommandError from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User - +from django.db.models import F logger = logging.getLogger(__name__) @@ -99,17 +99,83 @@ class Command(BaseCommand): display_as_str=True, ) + # TODO - add post processing step to add suborg city, state, etc. + # This needs to be done after because of execution order. + # However, we do not need to necessarily prompt the user in this case. + def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" portfolio, created = self.create_portfolio(federal_agency) + suborganizations = Suborganization.objects.none() + domains = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) + domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( + status__in=[ + DomainRequest.DomainRequestStatus.STARTED, + DomainRequest.DomainRequestStatus.INELIGIBLE, + DomainRequest.DomainRequestStatus.REJECTED, + ] + ) if created: - self.create_suborganizations(portfolio, federal_agency) + suborganizations = self.create_suborganizations(portfolio, federal_agency) if parse_domains or both: - self.handle_portfolio_domains(portfolio, federal_agency) + domains = self.handle_portfolio_domains(portfolio, federal_agency, domains) if parse_requests or both: - self.handle_portfolio_requests(portfolio, federal_agency) + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency, domain_requests) + + # Post process steps + # Add suborg info to created or existing suborgs. Get the refreshed queryset for each. + self.post_process_suborganization(suborganizations.all(), domains.all(), domain_requests.all()) + + def post_process_suborganization(self, suborganizations, domains, requests): + # Exclude domains and requests where the org name is the same, + # and where we are missing some crucial information. + domains = domains.exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name") + ).in_bulk("organization_name") + + requests = requests.exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name") + ).in_bulk("organization_name") + + for suborg in suborganizations: + domain = domains.get(suborg.name, None) + request = requests.get(suborg.name, None) + + # PRIORITY: + # 1. Domain info + # 2. Domain request requested suborg fields + # 3. Domain request normal fields + city = None + if domain and domain.city: + city = normalize_string(domain.city, lowercase=False) + elif request and request.suborganization_city: + city = normalize_string(request.suborganization_city, lowercase=False) + elif request and request.city: + city = normalize_string(request.city, lowercase=False) + + state_territory = None + if domain and domain.state_territory: + state_territory = domain.state_territory + elif request and request.suborganization_state_territory: + state_territory = request.suborganization_state_territory + elif request and request.state_territory: + state_territory = request.state_territory + + if city: + suborg.city = city + + if suborg: + suborg.state_territory = state_territory + + Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) def create_portfolio(self, federal_agency): """Creates a portfolio if it doesn't presently exist. @@ -200,20 +266,13 @@ class Command(BaseCommand): ) else: TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") + return new_suborgs - def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): + def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_requests): """ Associate portfolio with domain requests for a federal agency. Updates all relevant domain request records. """ - invalid_states = [ - DomainRequest.DomainRequestStatus.STARTED, - DomainRequest.DomainRequestStatus.INELIGIBLE, - DomainRequest.DomainRequestStatus.REJECTED, - ] - domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( - status__in=invalid_states - ) if not domain_requests.exists(): message = f""" Portfolio '{portfolio}' not added to domain requests: no valid records found. @@ -238,12 +297,14 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): + # Return a fresh copy of the queryset + return domain_requests.all() + + def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_infos): """ Associate portfolio with domains for a federal agency. Updates all relevant domain information records. """ - domain_infos = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) if not domain_infos.exists(): message = f""" Portfolio '{portfolio}' not added to domains: no valid records found. @@ -263,3 +324,6 @@ class Command(BaseCommand): DomainInformation.objects.bulk_update(domain_infos, ["portfolio", "sub_organization"]) message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + + # Return a fresh copy of the queryset + return domain_infos.all() diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py new file mode 100644 index 000000000..197c6fe46 --- /dev/null +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -0,0 +1,129 @@ +@ -0,0 +1,123 @@ +import logging +from django.core.management import BaseCommand +from registrar.models import Suborganization, DomainRequest, DomainInformation +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" + + def handle(self, **kwargs): + + # Step 1: delete duplicates + # Find duplicates + duplicates = {} + all_suborgs = Suborganization.objects.all() + + for suborg in all_suborgs: + # Normalize name by removing extra spaces and converting to lowercase + normalized_name = " ".join(suborg.name.split()).lower() + + # First occurrence of this name + if normalized_name not in duplicates: + duplicates[normalized_name] = { + "keep": suborg, + "delete": [] + } + continue + + # Compare with our current best + current_best = duplicates[normalized_name]["keep"] + + # Check if all other fields match. + # If they don't, we should inspect this record manually. + fields_to_compare = ["portfolio", "city", "state_territory"] + fields_match = all( + getattr(suborg, field) == getattr(current_best, field) + for field in fields_to_compare + ) + if not fields_match: + logger.warning( + f"{TerminalColors.YELLOW}" + f"\nSkipping potential duplicate: {suborg.name} (id: {suborg.id})" + f"\nData mismatch with {current_best.name} (id: {current_best.id})" + f"{TerminalColors.ENDC}" + ) + continue + + # Determine if new suborg is better than current best. + # The fewest spaces and most capitals wins. + new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") + new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) + # TODO + # Split into words and count properly capitalized first letters + # new_proper_caps = sum( + # 1 for word in suborg.name.split() + # if word and word[0].isupper() + # ) + # current_proper_caps = sum( + # 1 for word in current_best.name.split() + # if word and word[0].isupper() + # ) + # new_has_better_caps = new_proper_caps > current_proper_caps + + if new_has_fewer_spaces or new_has_more_capitals: + # New suborg is better - demote the old one to the delete list + duplicates[normalized_name]["delete"].append(current_best) + duplicates[normalized_name]["keep"] = suborg + else: + # If it is not better, just delete the old one + duplicates[normalized_name]["delete"].append(suborg) + + # Filter out entries without duplicates + duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} + if not duplicates: + logger.info(f"No duplicate suborganizations found.") + return + + # Show preview of changes + preview = "The following duplicates will be removed:\n" + for data in duplicates.values(): + best = data.get("keep") + preview += f"\nKeeping: '{best.name}' (id: {best.id})" + + for duplicate in data.get("delete"): + preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" + preview += "\n" + + # Get confirmation and execute deletions + if TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message=preview, + prompt_title="Clean up duplicate suborganizations?", + verify_message="*** WARNING: This will delete suborganizations! ***" + ): + try: + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] + + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + ids_to_delete = [ + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ] + + # Bulk delete all duplicates + delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() + logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") + + except Exception as e: + logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + + # Step 2: Add city, state, etc info to existing suborganizations. + From 14b6382a2dfbb221d62fabaf9d823f61bcbebdc4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:08:36 -0700 Subject: [PATCH 066/201] Add import --- .../commands/create_federal_portfolio.py | 10 ++++------ .../management/commands/patch_suborganizations.py | 15 ++++++++++----- src/registrar/models/utility/generic_helper.py | 9 +++++++++ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index f3debba56..37bd4765d 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -7,6 +7,8 @@ from registrar.management.commands.utility.terminal_helper import TerminalColors from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User from django.db.models import F +from registrar.models.utility.generic_helper import normalize_string + logger = logging.getLogger(__name__) @@ -99,10 +101,6 @@ class Command(BaseCommand): display_as_str=True, ) - # TODO - add post processing step to add suborg city, state, etc. - # This needs to be done after because of execution order. - # However, we do not need to necessarily prompt the user in this case. - def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" @@ -126,9 +124,9 @@ class Command(BaseCommand): # Post process steps # Add suborg info to created or existing suborgs. Get the refreshed queryset for each. - self.post_process_suborganization(suborganizations.all(), domains.all(), domain_requests.all()) + self.post_process_suborganization_fields(suborganizations.all(), domains.all(), domain_requests.all()) - def post_process_suborganization(self, suborganizations, domains, requests): + def post_process_suborganization_fields(self, suborganizations, domains, requests): # Exclude domains and requests where the org name is the same, # and where we are missing some crucial information. domains = domains.exclude( diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 197c6fe46..88ad611da 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -12,12 +12,19 @@ class Command(BaseCommand): help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" def handle(self, **kwargs): - + # Maybe we should just do these manually? + extra_records_to_delete = [ + "Assistant Secretary for Preparedness and Response Office of the Secretary", + "US Geological Survey", + "USDA/OC", + ] # Step 1: delete duplicates + self.delete_suborganization_duplicates() + + def delete_suborganization_duplicates(self, extra_records_to_delete): # Find duplicates duplicates = {} all_suborgs = Suborganization.objects.all() - for suborg in all_suborgs: # Normalize name by removing extra spaces and converting to lowercase normalized_name = " ".join(suborg.name.split()).lower() @@ -124,6 +131,4 @@ class Command(BaseCommand): except Exception as e: logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") - - # Step 2: Add city, state, etc info to existing suborganizations. - + diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 5e425f5a3..84dc28db1 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -343,3 +343,12 @@ def value_of_attribute(obj, attribute_name: str): if callable(value): value = value() return value + +def normalize_string(string_to_normalize, lowercase=True): + """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" + if not isinstance(string_to_normalize, str): + logger.error(f"normalize_string => {string_to_normalize} is not type str.") + return string_to_normalize + + new_string = " ".join(string_to_normalize.split()) + return new_string.lower() if lowercase else new_string From ae2e4c59995ec954403f9941be5092a87c69171d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:44:18 -0700 Subject: [PATCH 067/201] Ducttape --- .../commands/patch_suborganizations.py | 103 ++++++++++++------ 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 88ad611da..07fb94f3b 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -3,6 +3,7 @@ import logging from django.core.management import BaseCommand from registrar.models import Suborganization, DomainRequest, DomainInformation from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper +from registrar.models.utility.generic_helper import normalize_string logger = logging.getLogger(__name__) @@ -12,18 +13,58 @@ class Command(BaseCommand): help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" def handle(self, **kwargs): - # Maybe we should just do these manually? - extra_records_to_delete = [ + manual_records = [ "Assistant Secretary for Preparedness and Response Office of the Secretary", "US Geological Survey", "USDA/OC", ] - # Step 1: delete duplicates - self.delete_suborganization_duplicates() - - def delete_suborganization_duplicates(self, extra_records_to_delete): - # Find duplicates duplicates = {} + for record in Suborganization.objects.filter(name__in=manual_records): + if record.name: + norm_name = normalize_string(record.name) + duplicates[norm_name] = { + "keep": None, + "delete": [record] + } + + records_to_delete.update(self.handle_suborganization_duplicates()) + + # Get confirmation and execute deletions + if TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message=preview, + prompt_title="Clean up duplicate suborganizations?", + verify_message="*** WARNING: This will delete suborganizations! ***" + ): + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] + + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + records_to_delete = set( + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ) + try: + delete_count, _ = Suborganization.objects.filter(id__in=records_to_delete).delete() + logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") + except Exception as e: + logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + + + def handle_suborganization_duplicates(self, duplicates): + # Find duplicates all_suborgs = Suborganization.objects.all() for suborg in all_suborgs: # Normalize name by removing extra spaces and converting to lowercase @@ -103,32 +144,26 @@ class Command(BaseCommand): prompt_title="Clean up duplicate suborganizations?", verify_message="*** WARNING: This will delete suborganizations! ***" ): - try: - # Update all references to point to the right suborg before deletion - for record in duplicates.values(): - best_record = record.get("keep") - delete_ids = [dupe.id for dupe in record.get("delete")] - - # Update domain requests - DomainRequest.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - # Update domain information - DomainInformation.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - ids_to_delete = [ - dupe.id - for data in duplicates.values() - for dupe in data["delete"] - ] + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] - # Bulk delete all duplicates - delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() - logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") - - except Exception as e: - logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + records_to_delete = set( + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ) + return records_to_delete + else: + return set() From 964550403d537e7e39fc4cf57bc7f98030ae0c18 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 8 Jan 2025 17:45:24 -0500 Subject: [PATCH 068/201] init tooltips after ajax in domains table --- src/registrar/assets/src/js/getgov/helpers-uswds.js | 2 +- src/registrar/assets/src/js/getgov/table-base.js | 8 ++++++++ src/registrar/assets/src/js/getgov/table-domains.js | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/registrar/assets/src/js/getgov/helpers-uswds.js b/src/registrar/assets/src/js/getgov/helpers-uswds.js index 129d578b6..eec7b0818 100644 --- a/src/registrar/assets/src/js/getgov/helpers-uswds.js +++ b/src/registrar/assets/src/js/getgov/helpers-uswds.js @@ -4,7 +4,7 @@ * accessible directly in getgov.min.js * */ -export function initializeTooltips() { +export function uswdsInitializeTooltips() { function checkTooltip() { // Check that the tooltip library is loaded, and if not, wait and retry if (window.tooltip && typeof window.tooltip.init === 'function') { diff --git a/src/registrar/assets/src/js/getgov/table-base.js b/src/registrar/assets/src/js/getgov/table-base.js index e1d5c11ce..5f66b446f 100644 --- a/src/registrar/assets/src/js/getgov/table-base.js +++ b/src/registrar/assets/src/js/getgov/table-base.js @@ -375,6 +375,13 @@ export class BaseTable { */ loadModals(page, total, unfiltered_total) {} + /** + * Loads tooltips + sets up event listeners + * "Activates" the tooltips after the DOM updates + * Utilizes "uswdsInitializeTooltips" + */ + initializeTooltips() {} + /** * Allows us to customize the table display based on specific conditions and a user's permissions * Dynamically manages the visibility set up of columns, adding/removing headers @@ -471,6 +478,7 @@ export class BaseTable { this.initCheckboxListeners(); this.loadModals(data.page, data.total, data.unfiltered_total); + this.initializeTooltips(); // Do not scroll on first page load if (scroll) diff --git a/src/registrar/assets/src/js/getgov/table-domains.js b/src/registrar/assets/src/js/getgov/table-domains.js index a6373a5c2..2a00f1f95 100644 --- a/src/registrar/assets/src/js/getgov/table-domains.js +++ b/src/registrar/assets/src/js/getgov/table-domains.js @@ -1,4 +1,5 @@ import { BaseTable } from './table-base.js'; +import { uswdsInitializeTooltips } from './helpers-uswds.js'; export class DomainsTable extends BaseTable { @@ -66,6 +67,9 @@ export class DomainsTable extends BaseTable { `; tbody.appendChild(row); } + initializeTooltips() { + uswdsInitializeTooltips(); + } } export function initDomainsTable() { From b048ff96de5565042e126ffd64cbdc3750a5face Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 8 Jan 2025 18:26:56 -0500 Subject: [PATCH 069/201] handle multiple domains in email notifications --- src/registrar/admin.py | 3 +- .../templates/emails/domain_invitation.txt | 41 +++++---- .../emails/domain_invitation_subject.txt | 2 +- src/registrar/utility/email_invitations.py | 50 +++++++---- src/registrar/views/domain.py | 5 +- src/registrar/views/portfolios.py | 85 ++++++++++++------- 6 files changed, 119 insertions(+), 67 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 0f0c76ee3..06731db38 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1480,8 +1480,9 @@ class DomainInvitationAdmin(ListHeaderAdmin): send_domain_invitation_email( email=requested_email, requestor=requestor, - domain=domain, + domains=domain, is_member_of_different_org=member_of_a_different_org, + requested_user=requested_user ) if requested_user is not None: # Domain Invitation creation for an existing User diff --git a/src/registrar/templates/emails/domain_invitation.txt b/src/registrar/templates/emails/domain_invitation.txt index 068040205..4959f7c23 100644 --- a/src/registrar/templates/emails/domain_invitation.txt +++ b/src/registrar/templates/emails/domain_invitation.txt @@ -1,36 +1,46 @@ {% autoescape off %}{# In a text file, we don't want to have HTML entities escaped #} -Hi. +{% if requested_user and requested_user.first_name %} +Hi, {{ requested_user.first_name }}. +{% else %} +Hi, +{% endif %} -{{ requestor_email }} has added you as a manager on {{ domain.name }}. +{{ requestor_email }} has invited you to manage: +{% for domain in domains %} +{{ domain.name }} +{% endfor %} -You can manage this domain on the .gov registrar . +To manage domain information, visit the .gov registrar . ---------------------------------------------------------------- +{% if not requested_user %} YOU NEED A LOGIN.GOV ACCOUNT -You’ll need a Login.gov account to manage your .gov domain. Login.gov provides -a simple and secure process for signing in to many government services with one -account. +You’ll need a Login.gov account to access the .gov registrar. That account needs to be +associated with the following email address: {{ invitee_email_address }} -If you don’t already have one, follow these steps to create your -Login.gov account . +Login.gov provides a simple and secure process for signing in to many government +services with one account. If you don’t already have one, follow these steps to create +your Login.gov account . +{% endif %} DOMAIN MANAGEMENT -As a .gov domain manager, you can add or update information about your domain. -You’ll also serve as a contact for your .gov domain. Please keep your contact -information updated. +As a .gov domain manager, you can add or update information like name servers. You’ll +also serve as a contact for the domains you manage. Please keep your contact +information updated. Learn more about domain management . SOMETHING WRONG? -If you’re not affiliated with {{ domain.name }} or think you received this -message in error, reply to this email. +If you’re not affiliated with the .gov domains mentioned in this invitation or think you +received this message in error, reply to this email. THANK YOU -.Gov helps the public identify official, trusted information. Thank you for using a .gov domain. +.Gov helps the public identify official, trusted information. Thank you for using a .gov +domain. ---------------------------------------------------------------- @@ -38,5 +48,6 @@ The .gov team Contact us: Learn about .gov -The .gov registry is a part of the Cybersecurity and Infrastructure Security Agency (CISA) +The .gov registry is a part of the Cybersecurity and Infrastructure Security Agency +(CISA) {% endautoescape %} diff --git a/src/registrar/templates/emails/domain_invitation_subject.txt b/src/registrar/templates/emails/domain_invitation_subject.txt index 319b80176..9663346d0 100644 --- a/src/registrar/templates/emails/domain_invitation_subject.txt +++ b/src/registrar/templates/emails/domain_invitation_subject.txt @@ -1 +1 @@ -You’ve been added to a .gov domain \ No newline at end of file +You've been invited to manage {% if domains|length > 1 %}.gov domains{% else %}{{ domains.0.name }}{% endif %} \ No newline at end of file diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 9455c5927..630da7ce5 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -1,5 +1,7 @@ from django.conf import settings from registrar.models import DomainInvitation +from registrar.models.domain import Domain +from registrar.models.user import User from registrar.utility.errors import ( AlreadyDomainInvitedError, AlreadyDomainManagerError, @@ -13,7 +15,7 @@ import logging logger = logging.getLogger(__name__) -def send_domain_invitation_email(email: str, requestor, domain, is_member_of_different_org): +def send_domain_invitation_email(email: str, requestor, domains: Domain | list[Domain], is_member_of_different_org, requested_user=None): """ Sends a domain invitation email to the specified address. @@ -22,8 +24,9 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif Args: email (str): Email address of the recipient. requestor (User): The user initiating the invitation. - domain (Domain): The domain object for which the invitation is being sent. + domains (Domain or list of Domain): The domain objects for which the invitation is being sent. is_member_of_different_org (bool): if an email belongs to a different org + requested_user (User | None): The recipient if the email belongs to a user in the registrar Raises: MissingEmailError: If the requestor has no email associated with their account. @@ -32,13 +35,19 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif OutsideOrgMemberError: If the requested_user is part of a different organization. EmailSendingError: If there is an error while sending the email. """ + print('send_domain_invitation_email') + # Normalize domains + if isinstance(domains, Domain): + domains = [domains] + # Default email address for staff requestor_email = settings.DEFAULT_FROM_EMAIL # Check if the requestor is staff and has an email if not requestor.is_staff: if not requestor.email or requestor.email.strip() == "": - raise MissingEmailError(email=email, domain=domain) + domain_names = ", ".join([domain.name for domain in domains]) + raise MissingEmailError(email=email, domain=domain_names) else: requestor_email = requestor.email @@ -51,18 +60,19 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif ): raise OutsideOrgMemberError - # Check for an existing invitation - try: - invite = DomainInvitation.objects.get(email=email, domain=domain) - if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: - raise AlreadyDomainManagerError(email) - elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: - invite.update_cancellation_status() - invite.save() - else: - raise AlreadyDomainInvitedError(email) - except DomainInvitation.DoesNotExist: - pass + # Check for an existing invitation for each domain + for domain in domains: + try: + invite = DomainInvitation.objects.get(email=email, domain=domain) + if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: + raise AlreadyDomainManagerError(email) + elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: + invite.update_cancellation_status() + invite.save() + else: + raise AlreadyDomainInvitedError(email) + except DomainInvitation.DoesNotExist: + pass # Send the email try: @@ -71,12 +81,18 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif "emails/domain_invitation_subject.txt", to_address=email, context={ - "domain": domain, + "domains": domains, "requestor_email": requestor_email, + "invitee_email_address": email, + "requested_user": requested_user, }, ) except EmailSendingError as err: - raise EmailSendingError(f"Could not send email invitation to {email} for domain {domain}.") from err + print('point of failure test') + domain_names = ", ".join([domain.name for domain in domains]) + raise EmailSendingError( + f"Could not send email invitation to {email} for domains: {domain_names}" + ) from err def send_portfolio_invitation_email(email: str, requestor, portfolio): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b8464464e..f7938a301 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -1204,7 +1204,7 @@ class DomainAddUserView(DomainFormBaseView): send_domain_invitation_email( email=email, requestor=requestor, - domain=self.object, + domains=self.object, is_member_of_different_org=member_of_different_org, ) DomainInvitation.objects.get_or_create(email=email, domain=self.object) @@ -1215,8 +1215,9 @@ class DomainAddUserView(DomainFormBaseView): send_domain_invitation_email( email=email, requestor=requestor, - domain=self.object, + domains=self.object, is_member_of_different_org=member_of_different_org, + requested_user=requested_user, ) UserDomainRole.objects.create( user=requested_user, diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 60b30ad60..0cca1280c 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -8,13 +8,14 @@ from django.utils.safestring import mark_safe from django.contrib import messages from registrar.forms import portfolio as portfolioForms from registrar.models import Portfolio, User +from registrar.models.domain import Domain from registrar.models.domain_invitation import DomainInvitation from registrar.models.portfolio_invitation import PortfolioInvitation from registrar.models.user_domain_role import UserDomainRole from registrar.models.user_portfolio_permission import UserPortfolioPermission from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from registrar.utility.email import EmailSendingError -from registrar.utility.email_invitations import send_portfolio_invitation_email +from registrar.utility.email_invitations import send_domain_invitation_email, send_portfolio_invitation_email from registrar.utility.errors import MissingEmailError from registrar.utility.enums import DefaultUserValues from registrar.views.utility.mixins import PortfolioMemberPermission @@ -33,6 +34,8 @@ from django.views.generic import View from django.views.generic.edit import FormMixin from django.db import IntegrityError +from registrar.views.utility.portfolio_helper import get_org_membership + logger = logging.getLogger(__name__) @@ -237,6 +240,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V removed_domains = request.POST.get("removed_domains") portfolio_permission = get_object_or_404(UserPortfolioPermission, pk=pk) member = portfolio_permission.user + portfolio = portfolio_permission.portfolio added_domain_ids = self._parse_domain_ids(added_domains, "added domains") if added_domain_ids is None: @@ -248,7 +252,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V if added_domain_ids or removed_domain_ids: try: - self._process_added_domains(added_domain_ids, member) + self._process_added_domains(added_domain_ids, member, request.user, portfolio) self._process_removed_domains(removed_domain_ids, member) messages.success(request, "The domain assignment changes have been saved.") return redirect(reverse("member-domains", kwargs={"pk": pk})) @@ -263,7 +267,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V except Exception as e: messages.error( request, - "An unexpected error occurred: {str(e)}. If the issue persists, " + f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) logger.error(f"An unexpected error occurred: {str(e)}") @@ -287,16 +291,26 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V logger.error(f"Invalid data for {domain_type}") return None - def _process_added_domains(self, added_domain_ids, member): + def _process_added_domains(self, added_domain_ids, member, requestor, portfolio): """ Processes added domains by bulk creating UserDomainRole instances. """ if added_domain_ids: + print('_process_added_domains') + added_domains = Domain.objects.filter(id__in=added_domain_ids) + member_of_a_different_org, _ = get_org_membership(portfolio, member.email, member) + send_domain_invitation_email( + email=member.email, + requestor=requestor, + domains=added_domains, + is_member_of_different_org=member_of_a_different_org, + requested_user=member, + ) # Bulk create UserDomainRole instances for added domains UserDomainRole.objects.bulk_create( [ - UserDomainRole(domain_id=domain_id, user=member, role=UserDomainRole.Roles.MANAGER) - for domain_id in added_domain_ids + UserDomainRole(domain=domain, user=member, role=UserDomainRole.Roles.MANAGER) + for domain in added_domains ], ignore_conflicts=True, # Avoid duplicate entries ) @@ -443,6 +457,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission removed_domains = request.POST.get("removed_domains") portfolio_invitation = get_object_or_404(PortfolioInvitation, pk=pk) email = portfolio_invitation.email + portfolio = portfolio_invitation.portfolio added_domain_ids = self._parse_domain_ids(added_domains, "added domains") if added_domain_ids is None: @@ -454,7 +469,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission if added_domain_ids or removed_domain_ids: try: - self._process_added_domains(added_domain_ids, email) + self._process_added_domains(added_domain_ids, email, request.user, portfolio) self._process_removed_domains(removed_domain_ids, email) messages.success(request, "The domain assignment changes have been saved.") return redirect(reverse("invitedmember-domains", kwargs={"pk": pk})) @@ -469,7 +484,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission except Exception as e: messages.error( request, - "An unexpected error occurred: {str(e)}. If the issue persists, " + f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) logger.error(f"An unexpected error occurred: {str(e)}.") @@ -493,34 +508,41 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission logger.error(f"Invalid data for {domain_type}.") return None - def _process_added_domains(self, added_domain_ids, email): + def _process_added_domains(self, added_domain_ids, email, requestor, portfolio): """ Processes added domain invitations by updating existing invitations or creating new ones. """ - if not added_domain_ids: - return - - # Update existing invitations from CANCELED to INVITED - existing_invitations = DomainInvitation.objects.filter(domain_id__in=added_domain_ids, email=email) - existing_invitations.update(status=DomainInvitation.DomainInvitationStatus.INVITED) - - # Determine which domains need new invitations - existing_domain_ids = existing_invitations.values_list("domain_id", flat=True) - new_domain_ids = set(added_domain_ids) - set(existing_domain_ids) - - # Bulk create new invitations - DomainInvitation.objects.bulk_create( - [ - DomainInvitation( - domain_id=domain_id, + if added_domain_ids: + added_domains = Domain.objects.filter(id__in=added_domain_ids) + member_of_a_different_org, _ = get_org_membership(portfolio, email, None) + send_domain_invitation_email( email=email, - status=DomainInvitation.DomainInvitationStatus.INVITED, + requestor=requestor, + domains=added_domains, + is_member_of_different_org=member_of_a_different_org, ) - for domain_id in new_domain_ids - ] - ) + # Update existing invitations from CANCELED to INVITED + existing_invitations = DomainInvitation.objects.filter(domain__in=added_domains, email=email) + existing_invitations.update(status=DomainInvitation.DomainInvitationStatus.INVITED) + + # Determine which domains need new invitations + existing_domain_ids = existing_invitations.values_list("domain_id", flat=True) + new_domain_ids = set(added_domain_ids) - set(existing_domain_ids) + + # Bulk create new invitations + DomainInvitation.objects.bulk_create( + [ + DomainInvitation( + domain_id=domain_id, + email=email, + status=DomainInvitation.DomainInvitationStatus.INVITED, + ) + for domain_id in new_domain_ids + ] + ) + def _process_removed_domains(self, removed_domain_ids, email): """ Processes removed domain invitations by updating their status to CANCELED. @@ -755,8 +777,9 @@ class PortfolioAddMemberView(PortfolioMembersPermissionView, FormMixin): if not requested_user or not permission_exists: send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) portfolio_invitation = form.save() - portfolio_invitation.retrieve() - portfolio_invitation.save() + if requested_user is not None: + portfolio_invitation.retrieve() + portfolio_invitation.save() messages.success(self.request, f"{requested_email} has been invited.") else: if permission_exists: From acd20e89141f7a581fdb62281fc3d5c082d3b780 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 8 Jan 2025 21:16:01 -0800 Subject: [PATCH 070/201] Fix uncheck error --- src/registrar/templates/domain_renewal.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html index bf529a04d..7867bfce8 100644 --- a/src/registrar/templates/domain_renewal.html +++ b/src/registrar/templates/domain_renewal.html @@ -103,7 +103,9 @@ class="usa-checkbox__input" id="renewal-checkbox" type="checkbox" - name="{{ form.is_policy_acknowledged.name }}"> + name="{{ form.is_policy_acknowledged.name }}" + {% if form.is_policy_acknowledged.value %}checked{% endif %} + >
        {% url 'user-profile' as url %} - {% include "includes/summary_item.html" with title='Your Contact Information' value=user edit_link=url editable=is_editable contact='true' %} + {% include "includes/summary_item.html" with title='Your Contact Information' value=request.user edit_link=url editable=is_editable contact='true' %} diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 4b26caa5f..629d1bf03 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -311,24 +311,11 @@ class DomainView(DomainBaseView): self._update_session_with_domain() -class DomainRenewalView(DomainBaseView): +class DomainRenewalView(DomainView): """Domain detail overview page.""" template_name = "domain_renewal.html" - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - - default_emails = [DefaultEmail.PUBLIC_CONTACT_DEFAULT.value, DefaultEmail.LEGACY_DEFAULT.value] - context["hidden_security_emails"] = default_emails - - security_email = self.object.get_security_email() - user = self.request.user - if security_email is None or security_email in default_emails: - context["security_email"] = None - context["user"] = user - return context - def can_access_domain_via_portfolio(self, pk): """Most views should not allow permission to portfolio users. If particular views allow permissions, they will need to override @@ -341,28 +328,6 @@ class DomainRenewalView(DomainBaseView): return True return False - def in_editable_state(self, pk): - """Override in_editable_state from DomainPermission - Allow detail page to be viewable""" - - requested_domain = None - if Domain.objects.filter(id=pk).exists(): - requested_domain = Domain.objects.get(id=pk) - - # return true if the domain exists, this will allow the detail page to load - if requested_domain: - return True - return False - - def _get_domain(self, request): - """ - override get_domain for this view so that domain overview - always resets the cache for the domain object - """ - self.session = request.session - self.object = self.get_object() - self._update_session_with_domain() - def post(self, request, pk): domain = get_object_or_404(Domain, id=pk) From 6b2123d0f3dd7733b9a90b6192912023e480f4de Mon Sep 17 00:00:00 2001 From: asaki222 Date: Fri, 10 Jan 2025 11:44:04 -0500 Subject: [PATCH 114/201] reformatted and updated test --- src/registrar/tests/test_views_domain.py | 2 +- src/registrar/views/domain.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index d92da17dd..56bfe7306 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -607,7 +607,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): renewal_page = self.app.get(reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id})) # Verify we see "Your contact information" on the renewal form - self.assertContains(renewal_page, "Your contact information") + self.assertContains(renewal_page, "Your Contact Information") # Verify that the "Edit" button for Your contact is there and links to correct URL edit_button_url = reverse("user-profile") diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index bee59e4a9..629d1bf03 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -315,7 +315,7 @@ class DomainRenewalView(DomainView): """Domain detail overview page.""" template_name = "domain_renewal.html" - + def can_access_domain_via_portfolio(self, pk): """Most views should not allow permission to portfolio users. If particular views allow permissions, they will need to override From 6e1b352d4f4a2f11b1fd1f6cbc6b4e394a0121b4 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 10 Jan 2025 09:36:57 -0800 Subject: [PATCH 115/201] Fix your contact info title --- src/registrar/templates/domain_renewal.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html index 0e72568cc..f81389efb 100644 --- a/src/registrar/templates/domain_renewal.html +++ b/src/registrar/templates/domain_renewal.html @@ -49,7 +49,7 @@
        {% url 'user-profile' as url %} - {% include "includes/summary_item.html" with title='Your Contact Information' value=request.user edit_link=url editable=is_editable contact='true' %} + {% include "includes/summary_item.html" with title='Your contact information' value=request.user edit_link=url editable=is_editable contact='true' %} {% if analyst_action != 'edit' or analyst_action_location != domain.pk %} {% if is_portfolio_user and not is_domain_manager %} From 0ddb3a2ca7cf6894ae777c901026febda6f3973f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:46:58 -0700 Subject: [PATCH 116/201] Fix linter C901 error --- .../commands/patch_suborganizations.py | 107 +++++++++--------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 80676d6d2..98ff1e36f 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -27,57 +27,10 @@ class Command(BaseCommand): normalize_string("USDA/ARS/NAL"): {"replace_with": "USDA, ARS, NAL"}, } - # First: Group all suborganization names by their "normalized" names (finding duplicates). - # Returns a dict that looks like this: - # { - # "amtrak": [, , ], - # "usda/oc": [], - # ...etc - # } - # - name_groups = {} - for suborg in Suborganization.objects.all(): - normalized_name = normalize_string(suborg.name) - if normalized_name not in name_groups: - name_groups[normalized_name] = [] - name_groups[normalized_name].append(suborg) - - # Second: find the record we should keep, and the records we should delete - # Returns a dict that looks like this: - # { - # "amtrak": { - # "keep": - # "delete": [, ] - # }, - # "usda/oc": { - # "keep": , - # "delete": [] - # }, - # ...etc - # } - records_to_prune = {} - for normalized_name, duplicate_suborgs in name_groups.items(): - # Delete data from our preset list - if normalized_name in extra_records_to_prune: - # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. - # This assumes that there is only one item in the name_group array (see usda/oc example). - # But this should be fine, given our data. - hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] - name_group = name_groups.get(normalize_string(hardcoded_record_name)) - keep = name_group[0] if name_group else None - records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} - # Delete duplicates (extra spaces or casing differences) - elif len(duplicate_suborgs) > 1: - # Pick the best record (fewest spaces, most leading capitals) - best_record = max( - duplicate_suborgs, - key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True)), - ) - records_to_prune[normalized_name] = { - "keep": best_record, - "delete": [s for s in duplicate_suborgs if s != best_record], - } - + # Second: loop through every Suborganization and return a dict of what to keep, and what to delete + # for each duplicate or "incorrect" record. We do this by pruning records with extra spaces or bad caps + # Note that "extra_records_to_prune" is just a manual mapping. + records_to_prune = self.get_records_to_prune(extra_records_to_prune) if len(records_to_prune) == 0: TerminalHelper.colorful_logger(logger.error, TerminalColors.FAIL, "No suborganizations to delete.") return @@ -126,3 +79,55 @@ class Command(BaseCommand): TerminalHelper.colorful_logger( logger.error, TerminalColors.FAIL, f"Failed to delete suborganizations: {str(e)}" ) + + def get_records_to_prune(self, extra_records_to_prune): + """Maps all suborgs into a dictionary with a record to keep, and an array of records to delete.""" + # First: Group all suborganization names by their "normalized" names (finding duplicates). + # Returns a dict that looks like this: + # { + # "amtrak": [, , ], + # "usda/oc": [], + # ...etc + # } + # + name_groups = {} + for suborg in Suborganization.objects.all(): + normalized_name = normalize_string(suborg.name) + name_groups.setdefault(normalized_name, []).append(suborg) + + # Second: find the record we should keep, and the records we should delete + # Returns a dict that looks like this: + # { + # "amtrak": { + # "keep": + # "delete": [, ] + # }, + # "usda/oc": { + # "keep": , + # "delete": [] + # }, + # ...etc + # } + records_to_prune = {} + for normalized_name, duplicate_suborgs in name_groups.items(): + # Delete data from our preset list + if normalized_name in extra_records_to_prune: + # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. + # This assumes that there is only one item in the name_group array (see usda/oc example). + # But this should be fine, given our data. + hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] + name_group = name_groups.get(normalize_string(hardcoded_record_name)) + keep = name_group[0] if name_group else None + records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} + # Delete duplicates (extra spaces or casing differences) + elif len(duplicate_suborgs) > 1: + # Pick the best record (fewest spaces, most leading capitals) + best_record = max( + duplicate_suborgs, + key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True)), + ) + records_to_prune[normalized_name] = { + "keep": best_record, + "delete": [s for s in duplicate_suborgs if s != best_record], + } + return records_to_prune From 25ba5b2a5172c4f7e4cd79eac25663ec0a80b4a6 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 10 Jan 2025 09:56:58 -0800 Subject: [PATCH 117/201] Fix test capitalization --- src/registrar/tests/test_views_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 56bfe7306..d92da17dd 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -607,7 +607,7 @@ class TestDomainDetailDomainRenewal(TestDomainOverview): renewal_page = self.app.get(reverse("domain-renewal", kwargs={"pk": self.domain_with_ip.id})) # Verify we see "Your contact information" on the renewal form - self.assertContains(renewal_page, "Your Contact Information") + self.assertContains(renewal_page, "Your contact information") # Verify that the "Edit" button for Your contact is there and links to correct URL edit_button_url = reverse("user-profile") From 19114c7e7214474e2afcb49a57140846e40640c6 Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Fri, 10 Jan 2025 11:02:31 -0800 Subject: [PATCH 118/201] Fix checkbox issue --- src/registrar/templates/domain_renewal.html | 7 +++---- src/registrar/views/domain.py | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/registrar/templates/domain_renewal.html b/src/registrar/templates/domain_renewal.html index f81389efb..02c9666e3 100644 --- a/src/registrar/templates/domain_renewal.html +++ b/src/registrar/templates/domain_renewal.html @@ -94,14 +94,13 @@
        {% csrf_token %}
        - {% if form.is_policy_acknowledged.errors %} -

        {{ form.is_policy_acknowledged.errors|join:" " }}

        - {% endif %} +
        - + comboBoxEl.insertAdjacentHTML("beforeend", Sanitizer.escapeHTML` - +   @@ -1356,8 +1360,12 @@ const displayList = el => { for (let i = 0, len = selectEl.options.length; i < len; i += 1) { const optionEl = selectEl.options[i]; const optionId = `${listOptionBaseId}${options.length}`; - if (optionEl.value && (disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { - if (selectEl.value && optionEl.value === selectEl.value) { + // DOTGOV: modified combobox to allow for options with blank value + //if (optionEl.value && (disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { + if ((disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { + // DOTGOV: modified combobox to allow blank option value selections to be considered selected + //if (selectEl.value && optionEl.value === selectEl.value) { + if (selectEl.value && optionEl.value === selectEl.value || (!selectEl.value && !optionEl.value)) { selectedItemId = optionId; } if (disableFiltering && !firstFoundId && regex.test(optionEl.text)) { diff --git a/src/registrar/assets/src/js/getgov/combobox.js b/src/registrar/assets/src/js/getgov/combobox.js index 36b7aa0ad..95c5ff7e8 100644 --- a/src/registrar/assets/src/js/getgov/combobox.js +++ b/src/registrar/assets/src/js/getgov/combobox.js @@ -28,19 +28,6 @@ export function loadInitialValuesForComboBoxes() { // Override the default clear button behavior such that it no longer clears the input, // it just resets to the data-initial-value. // Due to the nature of how uswds works, this is slightly hacky. - // Use a MutationObserver to watch for changes in the dropdown list - const dropdownList = comboBox.querySelector(`#${input.id}--list`); - const observer = new MutationObserver(function(mutations) { - mutations.forEach(function(mutation) { - if (mutation.type === "childList") { - addBlankOption(clearInputButton, dropdownList, initialValue); - } - }); - }); - - // Configure the observer to watch for changes in the dropdown list - const config = { childList: true, subtree: true }; - observer.observe(dropdownList, config); // Input event listener to detect typing input.addEventListener("input", () => { @@ -87,27 +74,4 @@ export function loadInitialValuesForComboBoxes() { showElement(clearInputButton) } } - - function addBlankOption(clearInputButton, dropdownList, initialValue) { - if (dropdownList && !dropdownList.querySelector('[data-value=""]') && !isTyping) { - const blankOption = document.createElement("li"); - blankOption.setAttribute("role", "option"); - blankOption.setAttribute("data-value", ""); - blankOption.classList.add("usa-combo-box__list-option"); - if (!initialValue){ - blankOption.classList.add("usa-combo-box__list-option--selected") - } - blankOption.textContent = "⎯"; - - dropdownList.insertBefore(blankOption, dropdownList.firstChild); - blankOption.addEventListener("click", (e) => { - e.preventDefault(); - e.stopPropagation(); - overrideDefaultClearButton = false; - // Trigger the default clear behavior - clearInputButton.click(); - overrideDefaultClearButton = true; - }); - } - } } diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 7089409de..f577ed0ab 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -164,6 +164,7 @@ class DomainSuborganizationForm(forms.ModelForm): sub_organization = forms.ModelChoiceField( label="Suborganization name", queryset=Suborganization.objects.none(), + empty_label="⎯ (No suborganization)", required=False, widget=ComboboxWidget, ) @@ -468,12 +469,11 @@ class DomainOrgNameAddressForm(forms.ModelForm): state_territory = forms.ChoiceField( label="State, territory, or military post", required=True, - choices=DomainInformation.StateTerritoryChoices.choices, - widget=ComboboxWidget( - attrs={ - "required": True, - } - ), + choices=[("", "--Select--")] + DomainInformation.StateTerritoryChoices.choices, + error_messages={ + "required": ("Select the state, territory, or military post where your organization is located.") + }, + widget=ComboboxWidget(), ) class Meta: @@ -493,9 +493,6 @@ class DomainOrgNameAddressForm(forms.ModelForm): "organization_name": {"required": "Enter the name of your organization."}, "address_line1": {"required": "Enter the street address of your organization."}, "city": {"required": "Enter the city where your organization is located."}, - "state_territory": { - "required": "Select the state, territory, or military post where your organization is located." - }, } widgets = { "organization_name": forms.TextInput, diff --git a/src/registrar/forms/portfolio.py b/src/registrar/forms/portfolio.py index 4e2e7bdf1..13d956fe4 100644 --- a/src/registrar/forms/portfolio.py +++ b/src/registrar/forms/portfolio.py @@ -37,7 +37,10 @@ class PortfolioOrgAddressForm(forms.ModelForm): state_territory = forms.ChoiceField( label="State, territory, or military post", required=True, - choices=DomainInformation.StateTerritoryChoices.choices, + choices=[("", "--Select--")] + DomainInformation.StateTerritoryChoices.choices, + error_messages={ + "required": ("Select the state, territory, or military post where your organization is located.") + }, widget=ComboboxWidget, ) @@ -54,9 +57,6 @@ class PortfolioOrgAddressForm(forms.ModelForm): error_messages = { "address_line1": {"required": "Enter the street address of your organization."}, "city": {"required": "Enter the city where your organization is located."}, - "state_territory": { - "required": "Select the state, territory, or military post where your organization is located." - }, "zipcode": {"required": "Enter a 5-digit or 9-digit zip code, like 12345 or 12345-6789."}, } widgets = { From 1ca52b49bbe719f4c4b1cf101df1f20ce038f06e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Jan 2025 19:23:29 -0500 Subject: [PATCH 152/201] additional edits to uswds and to combobox.js to accomodate for empty values in options --- src/registrar/assets/js/uswds-edited.js | 53 +++++++++++++------ .../assets/src/js/getgov/combobox.js | 2 +- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/registrar/assets/js/uswds-edited.js b/src/registrar/assets/js/uswds-edited.js index b597f2d2b..590033a87 100644 --- a/src/registrar/assets/js/uswds-edited.js +++ b/src/registrar/assets/js/uswds-edited.js @@ -1037,7 +1037,7 @@ const noop = () => {}; * @param {string} value The new value of the element */ const changeElementValue = function (el) { - let value = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : ""; + let value = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : ""; const elementToChange = el; elementToChange.value = value; const event = new CustomEvent("change", { @@ -1167,13 +1167,21 @@ const enhanceComboBox = _comboBoxEl => { placeholder }); } - if (defaultValue) { - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if (optionEl.value === defaultValue) { - selectedOption = optionEl; - break; - } + // DOTGOV - allowing for defaultValue to be empty + //if (defaultValue) { + // for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + // const optionEl = selectEl.options[i]; + // if (optionEl.value === defaultValue) { + // selectedOption = optionEl; + // break; + // } + // } + //} + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if ((optionEl.value === defaultValue) || (!optionEl.value && !defaultValue)) { + selectedOption = optionEl; + break; } } @@ -1500,16 +1508,27 @@ const resetSelection = el => { } = getComboBoxContext(el); const selectValue = selectEl.value; const inputValue = (inputEl.value || "").toLowerCase(); - if (selectValue) { - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if (optionEl.value === selectValue) { - if (inputValue !== optionEl.text) { - changeElementValue(inputEl, optionEl.text); - } - comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); - return; + // DOTGOV - allow for option value to be empty string + //if (selectValue) { + // for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + // const optionEl = selectEl.options[i]; + // if (optionEl.value === selectValue) { + // if (inputValue !== optionEl.text) { + // changeElementValue(inputEl, optionEl.text); + // } + // comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); + // return; + // } + // } + //} + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if ((!selectValue && !optionEl.value) || optionEl.value === selectValue) { + if (inputValue !== optionEl.text) { + changeElementValue(inputEl, optionEl.text); } + comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); + return; } } if (inputValue) { diff --git a/src/registrar/assets/src/js/getgov/combobox.js b/src/registrar/assets/src/js/getgov/combobox.js index 95c5ff7e8..e0ecc92ad 100644 --- a/src/registrar/assets/src/js/getgov/combobox.js +++ b/src/registrar/assets/src/js/getgov/combobox.js @@ -48,7 +48,7 @@ export function loadInitialValuesForComboBoxes() { // Change the default input behaviour - have it reset to the data default instead clearInputButton.addEventListener("click", (e) => { - if (overrideDefaultClearButton && initialValue) { + if (overrideDefaultClearButton) { e.preventDefault(); e.stopPropagation(); input.click(); From 24e0243df3004f91ca2f99722ff90f1385ee4180 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Jan 2025 20:03:58 -0500 Subject: [PATCH 153/201] fixed bug with portfolio requesting entity --- src/registrar/assets/src/js/getgov/requesting-entity.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/requesting-entity.js b/src/registrar/assets/src/js/getgov/requesting-entity.js index 3bcdcd35c..e784419b4 100644 --- a/src/registrar/assets/src/js/getgov/requesting-entity.js +++ b/src/registrar/assets/src/js/getgov/requesting-entity.js @@ -9,15 +9,16 @@ export function handleRequestingEntityFieldset() { const formPrefix = "portfolio_requesting_entity"; const radioFieldset = document.getElementById(`id_${formPrefix}-requesting_entity_is_suborganization__fieldset`); const radios = radioFieldset?.querySelectorAll(`input[name="${formPrefix}-requesting_entity_is_suborganization"]`); - const select = document.getElementById(`id_${formPrefix}-sub_organization`); - const selectParent = select?.parentElement; + const input = document.getElementById(`id_${formPrefix}-sub_organization`); + const inputGrandParent = input?.parentElement?.parentElement; + const select = input?.previousElementSibling; const suborgContainer = document.getElementById("suborganization-container"); const suborgDetailsContainer = document.getElementById("suborganization-container__details"); const suborgAddtlInstruction = document.getElementById("suborganization-addtl-instruction"); const subOrgCreateNewOption = document.getElementById("option-to-add-suborg")?.value; // Make sure all crucial page elements exist before proceeding. // This more or less ensures that we are on the Requesting Entity page, and not elsewhere. - if (!radios || !select || !selectParent || !suborgContainer || !suborgDetailsContainer) return; + if (!radios || !input || !select || !inputGrandParent || !suborgContainer || !suborgDetailsContainer) return; // requestingSuborganization: This just broadly determines if they're requesting a suborg at all // requestingNewSuborganization: This variable determines if the user is trying to *create* a new suborganization or not. @@ -28,7 +29,7 @@ export function handleRequestingEntityFieldset() { if (radio != null) requestingSuborganization = radio?.checked && radio.value === "True"; requestingSuborganization ? showElement(suborgContainer) : hideElement(suborgContainer); if (select.options.length == 2) { // --Select-- and other are the only options - hideElement(selectParent); // Hide the select drop down and indicate requesting new suborg + hideElement(inputGrandParent); // Hide the combo box and indicate requesting new suborg hideElement(suborgAddtlInstruction); // Hide additional instruction related to the list requestingNewSuborganization.value = "True"; } else { From fabeddaa619a23192c517f831fd80a72c84e066f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Jan 2025 20:56:45 -0500 Subject: [PATCH 154/201] modified combobox to handle error class --- src/registrar/assets/js/uswds-edited.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/registrar/assets/js/uswds-edited.js b/src/registrar/assets/js/uswds-edited.js index 590033a87..60502050f 100644 --- a/src/registrar/assets/js/uswds-edited.js +++ b/src/registrar/assets/js/uswds-edited.js @@ -31,6 +31,7 @@ * - fixed bug in createHeaderButton which added newlines to header button tooltips * - modified combobox to allow for blank values in list * - modified aria label for X button in combobox to reflect modified behavior of button + * - modified combobox to handle error class */ if ("document" in window.self) { @@ -1223,6 +1224,11 @@ const enhanceComboBox = _comboBoxEl => { input.setAttribute("class", INPUT_CLASS); input.setAttribute("type", "text"); input.setAttribute("role", "combobox"); + // DOTGOV - handle error class for combobox + // Check if 'usa-input--error' exists in selectEl and add it to input if true + if (selectEl.classList.contains('usa-input--error')) { + input.classList.add('usa-input--error'); + } additionalAttributes.forEach(attr => Object.keys(attr).forEach(key => { const value = Sanitizer.escapeHTML`${attr[key]}`; input.setAttribute(key, value); From b5970ecb373f9d8f1f779d5670f999f507402e62 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 06:34:46 -0500 Subject: [PATCH 155/201] updates in response to PR suggestions --- src/registrar/admin.py | 3 +- .../templates/emails/domain_invitation.txt | 10 +---- src/registrar/tests/test_admin.py | 5 ++- src/registrar/utility/email_invitations.py | 5 ++- src/registrar/views/portfolios.py | 8 ++-- .../views/utility/invitation_helper.py | 39 +++++++++---------- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index cdcc0400e..eb4e1737a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1520,7 +1520,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): """ if not change: domain = obj.domain - domain_org = domain.domain_info.portfolio + domain_org = getattr(domain.domain_info, "portfolio", None) requested_email = obj.email # Look up a user with that email requested_user = get_requested_user(requested_email) @@ -1536,6 +1536,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): and not flag_is_active(request, "multiple_portfolios") and domain_org is not None and not member_of_this_org + and not member_of_a_different_org ): send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) portfolio_invitation, _ = PortfolioInvitation.objects.get_or_create( diff --git a/src/registrar/templates/emails/domain_invitation.txt b/src/registrar/templates/emails/domain_invitation.txt index 4959f7c23..a077bff26 100644 --- a/src/registrar/templates/emails/domain_invitation.txt +++ b/src/registrar/templates/emails/domain_invitation.txt @@ -1,15 +1,9 @@ {% autoescape off %}{# In a text file, we don't want to have HTML entities escaped #} -{% if requested_user and requested_user.first_name %} -Hi, {{ requested_user.first_name }}. -{% else %} -Hi, -{% endif %} +Hi,{% if requested_user and requested_user.first_name %} {{ requested_user.first_name }}.{% endif %} {{ requestor_email }} has invited you to manage: -{% for domain in domains %} -{{ domain.name }} +{% for domain in domains %}{{ domain.name }} {% endfor %} - To manage domain information, visit the .gov registrar . ---------------------------------------------------------------- diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index c9f7a9032..210a1a8e6 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -334,7 +334,10 @@ class TestDomainInvitationAdmin(TestCase): Should not send a out portfolio invitation. Should trigger success message for the domain invitation. Should retrieve the domain invitation. - Should not create a portfolio invitation.""" + Should not create a portfolio invitation. + + NOTE: This test may need to be reworked when the multiple_portfolio flag is fully fleshed out. + """ user = User.objects.create_user(email="test@example.com", username="username") diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 1491b65a5..48c796340 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -48,7 +48,10 @@ def normalize_domains(domains): def get_requestor_email(requestor, domains): - """Get the requestor's email or raise an error if it's missing.""" + """Get the requestor's email or raise an error if it's missing. + + If the requestor is staff, default email is returned. + """ if requestor.is_staff: return settings.DEFAULT_FROM_EMAIL diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index b9249ca6d..c4f60ca35 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -262,7 +262,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V "A database error occurred while saving changes. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error("A database error occurred while saving changes.") + logger.error("A database error occurred while saving changes.", exc_info=True) return redirect(reverse("member-domains-edit", kwargs={"pk": pk})) except Exception as e: messages.error( @@ -270,7 +270,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error(f"An unexpected error occurred: {str(e)}") + logger.error(f"An unexpected error occurred: {str(e)}", exc_info=True) return redirect(reverse("member-domains-edit", kwargs={"pk": pk})) else: messages.info(request, "No changes detected.") @@ -479,7 +479,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission "A database error occurred while saving changes. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error("A database error occurred while saving changes.") + logger.error("A database error occurred while saving changes.", exc_info=True) return redirect(reverse("invitedmember-domains-edit", kwargs={"pk": pk})) except Exception as e: messages.error( @@ -487,7 +487,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error(f"An unexpected error occurred: {str(e)}.") + logger.error(f"An unexpected error occurred: {str(e)}.", exc_info=True) return redirect(reverse("invitedmember-domains-edit", kwargs={"pk": pk})) else: messages.info(request, "No changes detected.") diff --git a/src/registrar/views/utility/invitation_helper.py b/src/registrar/views/utility/invitation_helper.py index 771300406..5c730d0c3 100644 --- a/src/registrar/views/utility/invitation_helper.py +++ b/src/registrar/views/utility/invitation_helper.py @@ -1,8 +1,6 @@ from django.contrib import messages from django.db import IntegrityError -from registrar.models.portfolio_invitation import PortfolioInvitation -from registrar.models.user import User -from registrar.models.user_portfolio_permission import UserPortfolioPermission +from registrar.models import PortfolioInvitation, User, UserPortfolioPermission from registrar.utility.email import EmailSendingError import logging @@ -20,32 +18,33 @@ logger = logging.getLogger(__name__) # any view, and were initially developed for domain.py, portfolios.py and admin.py -def get_org_membership(requestor_org, requested_email, requested_user): +def get_org_membership(org, email, user): """ - Verifies if an email belongs to a different organization as a member or invited member. - Verifies if an email belongs to this organization as a member or invited member. - User does not belong to any org can be deduced from the tuple returned. + Determines if an email/user belongs to a different organization or this organization + as either a member or an invited member. - Returns a tuple (member_of_a_different_org, member_of_this_org). + This function returns a tuple (member_of_a_different_org, member_of_this_org), + which provides: + - member_of_a_different_org: True if the user/email is associated with an organization other than the given org. + - member_of_this_org: True if the user/email is associated with the given org. + + Note: This implementation assumes single portfolio ownership for a user. + If the "multiple portfolios" feature is enabled, this logic may not account for + situations where a user or email belongs to multiple organizations. """ - # COMMENT: this code does not take into account when multiple portfolios flag is set to true - - # COMMENT: shouldn't this code be based on the organization of the domain, not the org - # of the requestor? requestor could have multiple portfolios - - # Check for existing permissions or invitations for the requested user - existing_org_permission = UserPortfolioPermission.objects.filter(user=requested_user).first() - existing_org_invitation = PortfolioInvitation.objects.filter(email=requested_email).first() + # Check for existing permissions or invitations for the user + existing_org_permission = UserPortfolioPermission.objects.filter(user=user).first() + existing_org_invitation = PortfolioInvitation.objects.filter(email=email).first() # Determine membership in a different organization - member_of_a_different_org = (existing_org_permission and existing_org_permission.portfolio != requestor_org) or ( - existing_org_invitation and existing_org_invitation.portfolio != requestor_org + member_of_a_different_org = (existing_org_permission and existing_org_permission.portfolio != org) or ( + existing_org_invitation and existing_org_invitation.portfolio != org ) # Determine membership in the same organization - member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == requestor_org) or ( - existing_org_invitation and existing_org_invitation.portfolio == requestor_org + member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == org) or ( + existing_org_invitation and existing_org_invitation.portfolio == org ) return member_of_a_different_org, member_of_this_org From f196dce6077c8c420c8cd517be28596327a74932 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 07:08:22 -0500 Subject: [PATCH 156/201] lint --- src/registrar/tests/test_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 210a1a8e6..2a7a52a13 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -335,7 +335,7 @@ class TestDomainInvitationAdmin(TestCase): Should trigger success message for the domain invitation. Should retrieve the domain invitation. Should not create a portfolio invitation. - + NOTE: This test may need to be reworked when the multiple_portfolio flag is fully fleshed out. """ From ac3286d7801fd165500f3b09415ed7153ae0c21c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 07:26:01 -0500 Subject: [PATCH 157/201] fixed bug in portfolio invitation change view query --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index eb4e1737a..2e1b15dfb 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1594,7 +1594,7 @@ class PortfolioInvitationAdmin(BaseInvitationAdmin): # Search search_fields = [ "email", - "portfolio__name", + "portfolio__organization_name", ] # Filters From 74ef30b52d222171daa01c1681bdc42c797ace92 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 12:44:47 -0500 Subject: [PATCH 158/201] fixed the 'other' problem in requesting entity form --- .../assets/src/js/getgov/requesting-entity.js | 5 ----- src/registrar/forms/domain_request_wizard.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/requesting-entity.js b/src/registrar/assets/src/js/getgov/requesting-entity.js index e784419b4..5f3be8c79 100644 --- a/src/registrar/assets/src/js/getgov/requesting-entity.js +++ b/src/registrar/assets/src/js/getgov/requesting-entity.js @@ -38,11 +38,6 @@ export function handleRequestingEntityFieldset() { requestingNewSuborganization.value === "True" ? showElement(suborgDetailsContainer) : hideElement(suborgDetailsContainer); } - // Add fake "other" option to sub_organization select - if (select && !Array.from(select.options).some(option => option.value === "other")) { - select.add(new Option(subOrgCreateNewOption, "other")); - } - if (requestingNewSuborganization.value === "True") { select.value = "other"; } diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 2365d323d..636a41760 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -63,13 +63,19 @@ class RequestingEntityForm(RegistrarForm): ) def __init__(self, *args, **kwargs): - """Override of init to add the suborganization queryset""" + """Override of init to add the suborganization queryset and 'other' option""" super().__init__(*args, **kwargs) if self.domain_request.portfolio: - self.fields["sub_organization"].queryset = Suborganization.objects.filter( - portfolio=self.domain_request.portfolio - ) + # Fetch the queryset for the portfolio + queryset = Suborganization.objects.filter(portfolio=self.domain_request.portfolio) + # set the queryset appropriately so that post can validate against queryset + self.fields["sub_organization"].queryset = queryset + + # Modify the choices to include "other" so that form can display options properly + self.fields["sub_organization"].choices = [("", "--Select--")] + [ + (obj.id, str(obj)) for obj in queryset + ] + [("other", "Other (enter your suborganization manually)")] def clean_sub_organization(self): """On suborganization clean, set the suborganization value to None if the user is requesting From c0f5dca8c1d7b514209556f2f85ed06acb0583d5 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 14:50:59 -0500 Subject: [PATCH 159/201] add email to domain managers on domain invitation --- .../emails/domain_manager_notification.txt | 43 +++++++++++++++++++ .../domain_manager_notification_subject.txt | 1 + src/registrar/utility/email_invitations.py | 37 ++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 src/registrar/templates/emails/domain_manager_notification.txt create mode 100644 src/registrar/templates/emails/domain_manager_notification_subject.txt diff --git a/src/registrar/templates/emails/domain_manager_notification.txt b/src/registrar/templates/emails/domain_manager_notification.txt new file mode 100644 index 000000000..aa8c6bf34 --- /dev/null +++ b/src/registrar/templates/emails/domain_manager_notification.txt @@ -0,0 +1,43 @@ +{% autoescape off %}{# In a text file, we don't want to have HTML entities escaped #} +Hi,{% if domain_manager and domain_manager.first_name %} {{ domain_manager.first_name }}.{% endif %} + +A domain manager was invited to {{ domain.name }}. +DOMAIN: {{ domain.name }} +INVITED BY: {{ requestor_email }} +INVITED ON: {{date}} +MANAGER INVITED: {{ invited_email_address }} + + +---------------------------------------------------------------- + + +NEXT STEPS + +The person who received the invitation will become a domain manager once they log in to the +.gov registrar. They'll need to access the registrar using a Login.gov account that's +associated with the invited email address. + +If you need to cancel this invitation or remove the domain manager (because they've already +logged in), you can do that by going to this domain in the .gov registrar . + + +WHY DID YOU RECEIVE THIS EMAIL? + +You’re listed as a domain manager for {{ domain.name }}, so you’ll receive a notification whenever +someone is invited to manage that domain. + +If you have questions or concerns, reach out to the person who sent the invitation or reply to this email. + + +THANK YOU +.Gov helps the public identify official, trusted information. Thank you for using a .gov domain. + +---------------------------------------------------------------- + +The .gov team +Contact us: +Learn about .gov + +The .gov registry is a part of the Cybersecurity and Infrastructure Security Agency +(CISA) +{% endautoescape %} diff --git a/src/registrar/templates/emails/domain_manager_notification_subject.txt b/src/registrar/templates/emails/domain_manager_notification_subject.txt new file mode 100644 index 000000000..0e9918de0 --- /dev/null +++ b/src/registrar/templates/emails/domain_manager_notification_subject.txt @@ -0,0 +1 @@ +A domain manager was invited to {{ domain.name }} \ No newline at end of file diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 48c796340..fda901fba 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -1,6 +1,8 @@ +from datetime import date from django.conf import settings from registrar.models import DomainInvitation from registrar.models.domain import Domain +from registrar.models.user_domain_role import UserDomainRole from registrar.utility.errors import ( AlreadyDomainInvitedError, AlreadyDomainManagerError, @@ -41,6 +43,39 @@ def send_domain_invitation_email( send_invitation_email(email, requestor_email, domains, requested_user) + # send emails to domain managers + for domain in domains: + send_emails_to_domain_managers( + email=email, + requestor_email=requestor_email, + domain=domain, + requested_user=requested_user, + ) + + +def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, requested_user=None): + # Get each domain manager from list + user_domain_roles = UserDomainRole.objects.filter(domain=domain) + for user_domain_role in user_domain_roles: + # Send email to each domain manager + user = user_domain_role.user + try: + send_templated_email( + "emails/domain_manager_notification.txt", + "emails/domain_manager_notification_subject.txt", + to_address=user.email, + context={ + "domain": domain, + "requestor_email": requestor_email, + "invited_email_address": email, + "domain_manager": user, + "date": date.today(), + }, + ) + except EmailSendingError as err: + domain_names = ", ".join([domain.name for domain in domains]) + raise EmailSendingError(f"Could not send email invitation to {email} for domains: {domain_names}") from err + def normalize_domains(domains): """Ensures domains is always a list.""" @@ -69,6 +104,8 @@ def validate_invitation(email, domains, requestor, is_member_of_different_org): for domain in domains: validate_existing_invitation(email, domain) + # NOTE: should we also be validating against existing user_domain_roles + def check_outside_org_membership(email, requestor, is_member_of_different_org): """Raise an error if the email belongs to a different organization.""" From ca136c650c710a14a17b9f3fd8501fee48a36b23 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 14:56:04 -0500 Subject: [PATCH 160/201] updated error message when EmailSendingError --- src/registrar/utility/email_invitations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index fda901fba..ba660499e 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -73,8 +73,7 @@ def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, }, ) except EmailSendingError as err: - domain_names = ", ".join([domain.name for domain in domains]) - raise EmailSendingError(f"Could not send email invitation to {email} for domains: {domain_names}") from err + raise EmailSendingError(f"Could not send email manager notification to {user.email} for domains: {domain.name}") from err def normalize_domains(domains): From 287638786dbdc4a47c405cfb3e10dc0c7a8115cb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 15 Jan 2025 15:03:24 -0500 Subject: [PATCH 161/201] informational alerts on change forms --- src/registrar/admin.py | 4 +++- .../admin/domain_invitation_change_form.html | 14 ++++++++++++++ .../django/admin/user_domain_role_change_form.html | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/registrar/templates/django/admin/domain_invitation_change_form.html create mode 100644 src/registrar/templates/django/admin/user_domain_role_change_form.html diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 1b6e2de6a..e89147b11 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1367,6 +1367,8 @@ class UserDomainRoleAdmin(ListHeaderAdmin, ImportExportModelAdmin): autocomplete_fields = ["user", "domain"] + change_form_template = "django/admin/user_domain_role_change_form.html" + # Fixes a bug where non-superusers are redirected to the main page def delete_view(self, request, object_id, extra_context=None): """Custom delete_view implementation that specifies redirect behaviour""" @@ -1500,7 +1502,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): autocomplete_fields = ["domain"] - change_form_template = "django/admin/email_clipboard_change_form.html" + change_form_template = "django/admin/domain_invitation_change_form.html" # Select domain invitations to change -> Domain invitations def changelist_view(self, request, extra_context=None): diff --git a/src/registrar/templates/django/admin/domain_invitation_change_form.html b/src/registrar/templates/django/admin/domain_invitation_change_form.html new file mode 100644 index 000000000..6ce6ed0d1 --- /dev/null +++ b/src/registrar/templates/django/admin/domain_invitation_change_form.html @@ -0,0 +1,14 @@ +{% extends 'django/admin/email_clipboard_change_form.html' %} +{% load custom_filters %} +{% load i18n static %} + +{% block content_subtitle %} +
        +
        +

        + If you add someone to a domain here, it will trigger emails to the invitee and all managers of the domain when you click "save." If you don't want to trigger those emails, use the User domain roles permissions table instead. +

        +
        +
        + {{ block.super }} +{% endblock %} diff --git a/src/registrar/templates/django/admin/user_domain_role_change_form.html b/src/registrar/templates/django/admin/user_domain_role_change_form.html new file mode 100644 index 000000000..200734ec1 --- /dev/null +++ b/src/registrar/templates/django/admin/user_domain_role_change_form.html @@ -0,0 +1,14 @@ +{% extends 'django/admin/email_clipboard_change_form.html' %} +{% load custom_filters %} +{% load i18n static %} + +{% block content_subtitle %} +
        +
        +

        + If you add someone to a domain here, it won't trigger any emails. To trigger emails, use the User Domain Role invitations table instead. +

        +
        +
        + {{ block.super }} +{% endblock %} From cb8494017acbc69c843f19edb727a0392338b3a3 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 15 Jan 2025 15:28:15 -0500 Subject: [PATCH 162/201] template tests --- .../admin/user_domain_role_change_form.html | 2 +- src/registrar/tests/test_admin.py | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/registrar/templates/django/admin/user_domain_role_change_form.html b/src/registrar/templates/django/admin/user_domain_role_change_form.html index 200734ec1..d8c298bc1 100644 --- a/src/registrar/templates/django/admin/user_domain_role_change_form.html +++ b/src/registrar/templates/django/admin/user_domain_role_change_form.html @@ -6,7 +6,7 @@

        - If you add someone to a domain here, it won't trigger any emails. To trigger emails, use the User Domain Role invitations table instead. + If you add someone to a domain here, it will not trigger any emails. To trigger emails, use the User Domain Role invitations table instead.

        diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 2a7a52a13..673057e20 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -165,6 +165,33 @@ class TestDomainInvitationAdmin(TestCase): response, "Domain invitations contain all individuals who have been invited to manage a .gov domain." ) self.assertContains(response, "Show more") + + @less_console_noise_decorator + def test_has_change_form_description(self): + """Tests if this model has a model description on the change form view""" + self.client.force_login(self.superuser) + + domain, _ = Domain.objects.get_or_create( + name="systemofadown.com" + ) + + domain_invitation, _ = DomainInvitation.objects.get_or_create( + email="toxicity@systemofadown.com", domain=domain + ) + + response = self.client.get( + "/admin/registrar/domaininvitation/{}/change/".format(domain_invitation.pk), + follow=True, + ) + + # Make sure that the page is loaded correctly + self.assertEqual(response.status_code, 200) + + # Test for a description snippet + self.assertContains( + response, + "If you add someone to a domain here, it will trigger emails to the invitee and all managers of the domain when you click", + ) @less_console_noise_decorator def test_get_filters(self): @@ -1956,6 +1983,33 @@ class TestUserDomainRoleAdmin(TestCase): response, "This table represents the managers who are assigned to each domain in the registrar" ) self.assertContains(response, "Show more") + + @less_console_noise_decorator + def test_has_change_form_description(self): + """Tests if this model has a model description on the change form view""" + self.client.force_login(self.superuser) + + domain, _ = Domain.objects.get_or_create( + name="systemofadown.com" + ) + + user_domain_role, _ = UserDomainRole.objects.get_or_create( + user=self.superuser, domain=domain, role=[UserDomainRole.Roles.MANAGER] + ) + + response = self.client.get( + "/admin/registrar/userdomainrole/{}/change/".format(user_domain_role.pk), + follow=True, + ) + + # Make sure that the page is loaded correctly + self.assertEqual(response.status_code, 200) + + # Test for a description snippet + self.assertContains( + response, + "If you add someone to a domain here, it will not trigger any emails.", + ) def test_domain_sortable(self): """Tests if the UserDomainrole sorts by domain correctly""" From 4a0dc40cee4e26d4f948afb4b76105a43ad35cc5 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 15 Jan 2025 16:00:03 -0500 Subject: [PATCH 163/201] fix mock_send_domain_email unit tests --- src/registrar/tests/test_views_domain.py | 43 +++++++++++++----------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index f13490312..a9de8d6e7 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -750,11 +750,12 @@ class TestDomainManagers(TestDomainOverview): response = self.client.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) self.assertContains(response, "Add a domain manager") - @boto3_mocking.patching @less_console_noise_decorator - def test_domain_user_add_form(self): + @patch("registrar.views.domain.send_domain_invitation_email") + def test_domain_user_add_form(self, mock_send_domain_email): """Adding an existing user works.""" get_user_model().objects.get_or_create(email="mayor@igorville.gov") + user = User.objects.filter(email="mayor@igorville.gov").first() add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] @@ -762,10 +763,11 @@ class TestDomainManagers(TestDomainOverview): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - mock_client = MockSESClient() - with boto3_mocking.clients.handler_for("sesv2", mock_client): - with less_console_noise(): - success_result = add_page.form.submit() + success_result = add_page.form.submit() + + mock_send_domain_email.assert_called_once_with( + email="mayor@igorville.gov", requestor=self.user, domains=self.domain, is_member_of_different_org=None, requested_user=user + ) self.assertEqual(success_result.status_code, 302) self.assertEqual( @@ -974,13 +976,13 @@ class TestDomainManagers(TestDomainOverview): success_page = success_result.follow() self.assertContains(success_page, "Failed to send email.") - @boto3_mocking.patching @less_console_noise_decorator - def test_domain_invitation_created(self): + @patch("registrar.views.domain.send_domain_invitation_email") + def test_domain_invitation_created(self, mock_send_domain_email): """Add user on a nonexistent email creates an invitation. Adding a non-existent user sends an email as a side-effect, so mock - out the boto3 SES email sending here. + out send_domain_invitation_email here. """ # make sure there is no user with this email email_address = "mayor@igorville.gov" @@ -993,10 +995,11 @@ class TestDomainManagers(TestDomainOverview): add_page.form["email"] = email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - mock_client = MockSESClient() - with boto3_mocking.clients.handler_for("sesv2", mock_client): - with less_console_noise(): - success_result = add_page.form.submit() + success_result = add_page.form.submit() + + mock_send_domain_email.assert_called_once_with( + email="mayor@igorville.gov", requestor=self.user, domains=self.domain, is_member_of_different_org=None + ) self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() @@ -1005,13 +1008,13 @@ class TestDomainManagers(TestDomainOverview): self.assertContains(success_page, "Cancel") # link to cancel invitation self.assertTrue(DomainInvitation.objects.filter(email=email_address).exists()) - @boto3_mocking.patching @less_console_noise_decorator - def test_domain_invitation_created_for_caps_email(self): + @patch("registrar.views.domain.send_domain_invitation_email") + def test_domain_invitation_created_for_caps_email(self, mock_send_domain_email): """Add user on a nonexistent email with CAPS creates an invitation to lowercase email. Adding a non-existent user sends an email as a side-effect, so mock - out the boto3 SES email sending here. + out send_domain_invitation_email here. """ # make sure there is no user with this email email_address = "mayor@igorville.gov" @@ -1025,9 +1028,11 @@ class TestDomainManagers(TestDomainOverview): add_page.form["email"] = caps_email_address self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - mock_client = MockSESClient() - with boto3_mocking.clients.handler_for("sesv2", mock_client): - success_result = add_page.form.submit() + success_result = add_page.form.submit() + + mock_send_domain_email.assert_called_once_with( + email="mayor@igorville.gov", requestor=self.user, domains=self.domain, is_member_of_different_org=None + ) self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() From c10a7738ca236a0c9572cde5a1cda8c6f8679359 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 16:27:08 -0500 Subject: [PATCH 164/201] tests and lint --- src/registrar/tests/test_email_invitations.py | 303 ++++++++++++++++++ src/registrar/utility/email_invitations.py | 8 +- 2 files changed, 307 insertions(+), 4 deletions(-) create mode 100644 src/registrar/tests/test_email_invitations.py diff --git a/src/registrar/tests/test_email_invitations.py b/src/registrar/tests/test_email_invitations.py new file mode 100644 index 000000000..6a0423f4d --- /dev/null +++ b/src/registrar/tests/test_email_invitations.py @@ -0,0 +1,303 @@ +import unittest +from unittest.mock import patch, MagicMock +from datetime import date +from registrar.utility.email import EmailSendingError +from registrar.utility.email_invitations import send_domain_invitation_email + + +class DomainInvitationEmail(unittest.TestCase): + + @patch("registrar.utility.email_invitations.send_templated_email") + @patch("registrar.utility.email_invitations.UserDomainRole.objects.filter") + @patch("registrar.utility.email_invitations.validate_invitation") + @patch("registrar.utility.email_invitations.get_requestor_email") + @patch("registrar.utility.email_invitations.send_invitation_email") + @patch("registrar.utility.email_invitations.normalize_domains") + def test_send_domain_invitation_email( + self, + mock_normalize_domains, + mock_send_invitation_email, + mock_get_requestor_email, + mock_validate_invitation, + mock_user_domain_role_filter, + mock_send_templated_email, + ): + """Test sending domain invitation email for one domain. + Should also send emails to manager of that domain. + """ + # Setup + mock_domain = MagicMock(name="domain1") + mock_domain.name = "example.com" + mock_normalize_domains.return_value = [mock_domain] + + mock_requestor = MagicMock() + mock_requestor_email = "requestor@example.com" + mock_get_requestor_email.return_value = mock_requestor_email + + mock_user1 = MagicMock() + mock_user1.email = "manager1@example.com" + + mock_user_domain_role_filter.return_value = [MagicMock(user=mock_user1)] + + email = "invitee@example.com" + is_member_of_different_org = False + + # Call the function + send_domain_invitation_email( + email=email, + requestor=mock_requestor, + domains=mock_domain, + is_member_of_different_org=is_member_of_different_org, + ) + + # Assertions + mock_normalize_domains.assert_called_once_with(mock_domain) + mock_get_requestor_email.assert_called_once_with(mock_requestor, [mock_domain]) + mock_validate_invitation.assert_called_once_with( + email, [mock_domain], mock_requestor, is_member_of_different_org + ) + mock_send_invitation_email.assert_called_once_with(email, mock_requestor_email, [mock_domain], None) + mock_user_domain_role_filter.assert_called_once_with(domain=mock_domain) + mock_send_templated_email.assert_called_once_with( + "emails/domain_manager_notification.txt", + "emails/domain_manager_notification_subject.txt", + to_address=mock_user1.email, + context={ + "domain": mock_domain, + "requestor_email": mock_requestor_email, + "invited_email_address": email, + "domain_manager": mock_user1, + "date": date.today(), + }, + ) + + @patch("registrar.utility.email_invitations.send_templated_email") + @patch("registrar.utility.email_invitations.UserDomainRole.objects.filter") + @patch("registrar.utility.email_invitations.validate_invitation") + @patch("registrar.utility.email_invitations.get_requestor_email") + @patch("registrar.utility.email_invitations.send_invitation_email") + @patch("registrar.utility.email_invitations.normalize_domains") + def test_send_domain_invitation_email_multiple_domains( + self, + mock_normalize_domains, + mock_send_invitation_email, + mock_get_requestor_email, + mock_validate_invitation, + mock_user_domain_role_filter, + mock_send_templated_email, + ): + """Test sending domain invitation email for multiple domains. + Should also send emails to managers of each domain. + """ + # Setup + # Create multiple mock domains + mock_domain1 = MagicMock(name="domain1") + mock_domain1.name = "example.com" + mock_domain2 = MagicMock(name="domain2") + mock_domain2.name = "example.org" + + mock_normalize_domains.return_value = [mock_domain1, mock_domain2] + + mock_requestor = MagicMock() + mock_requestor_email = "requestor@example.com" + mock_get_requestor_email.return_value = mock_requestor_email + + mock_user1 = MagicMock() + mock_user1.email = "manager1@example.com" + mock_user2 = MagicMock() + mock_user2.email = "manager2@example.com" + + # Configure domain roles for each domain + def filter_side_effect(domain): + if domain == mock_domain1: + return [MagicMock(user=mock_user1)] + elif domain == mock_domain2: + return [MagicMock(user=mock_user2)] + return [] + + mock_user_domain_role_filter.side_effect = filter_side_effect + + email = "invitee@example.com" + is_member_of_different_org = False + + # Call the function + send_domain_invitation_email( + email=email, + requestor=mock_requestor, + domains=[mock_domain1, mock_domain2], + is_member_of_different_org=is_member_of_different_org, + ) + + # Assertions + mock_normalize_domains.assert_called_once_with([mock_domain1, mock_domain2]) + mock_get_requestor_email.assert_called_once_with(mock_requestor, [mock_domain1, mock_domain2]) + mock_validate_invitation.assert_called_once_with( + email, [mock_domain1, mock_domain2], mock_requestor, is_member_of_different_org + ) + mock_send_invitation_email.assert_called_once_with( + email, mock_requestor_email, [mock_domain1, mock_domain2], None + ) + + # Check that domain manager emails were sent for both domains + mock_user_domain_role_filter.assert_any_call(domain=mock_domain1) + mock_user_domain_role_filter.assert_any_call(domain=mock_domain2) + + mock_send_templated_email.assert_any_call( + "emails/domain_manager_notification.txt", + "emails/domain_manager_notification_subject.txt", + to_address=mock_user1.email, + context={ + "domain": mock_domain1, + "requestor_email": mock_requestor_email, + "invited_email_address": email, + "domain_manager": mock_user1, + "date": date.today(), + }, + ) + mock_send_templated_email.assert_any_call( + "emails/domain_manager_notification.txt", + "emails/domain_manager_notification_subject.txt", + to_address=mock_user2.email, + context={ + "domain": mock_domain2, + "requestor_email": mock_requestor_email, + "invited_email_address": email, + "domain_manager": mock_user2, + "date": date.today(), + }, + ) + + # Verify the total number of calls to send_templated_email + self.assertEqual(mock_send_templated_email.call_count, 2) + + @patch("registrar.utility.email_invitations.validate_invitation") + def test_send_domain_invitation_email_raises_invite_validation_exception(self, mock_validate_invitation): + """Test sending domain invitation email for one domain and assert exception + when invite validation fails. + """ + # Setup + mock_validate_invitation.side_effect = ValueError("Validation failed") + email = "invitee@example.com" + requestor = MagicMock() + domain = MagicMock() + + # Call and assert exception + with self.assertRaises(ValueError) as context: + send_domain_invitation_email(email, requestor, domain, is_member_of_different_org=False) + + self.assertEqual(str(context.exception), "Validation failed") + mock_validate_invitation.assert_called_once() + + @patch("registrar.utility.email_invitations.get_requestor_email") + def test_send_domain_invitation_email_raises_get_requestor_email_exception(self, mock_get_requestor_email): + """Test sending domain invitation email for one domain and assert exception + when get_requestor_email fails. + """ + # Setup + mock_get_requestor_email.side_effect = ValueError("Validation failed") + email = "invitee@example.com" + requestor = MagicMock() + domain = MagicMock() + + # Call and assert exception + with self.assertRaises(ValueError) as context: + send_domain_invitation_email(email, requestor, domain, is_member_of_different_org=False) + + self.assertEqual(str(context.exception), "Validation failed") + mock_get_requestor_email.assert_called_once() + + @patch("registrar.utility.email_invitations.validate_invitation") + @patch("registrar.utility.email_invitations.get_requestor_email") + @patch("registrar.utility.email_invitations.send_invitation_email") + @patch("registrar.utility.email_invitations.normalize_domains") + def test_send_domain_invitation_email_raises_sending_email_exception( + self, + mock_normalize_domains, + mock_send_invitation_email, + mock_get_requestor_email, + mock_validate_invitation, + ): + """Test sending domain invitation email for one domain and assert exception + when send_invitation_email fails. + """ + # Setup + mock_domain = MagicMock(name="domain1") + mock_domain.name = "example.com" + mock_normalize_domains.return_value = [mock_domain] + + mock_requestor = MagicMock() + mock_requestor_email = "requestor@example.com" + mock_get_requestor_email.return_value = mock_requestor_email + + mock_user1 = MagicMock() + mock_user1.email = "manager1@example.com" + + email = "invitee@example.com" + is_member_of_different_org = False + + mock_send_invitation_email.side_effect = EmailSendingError("Error sending email") + + # Call and assert exception + with self.assertRaises(EmailSendingError) as context: + send_domain_invitation_email( + email=email, + requestor=mock_requestor, + domains=mock_domain, + is_member_of_different_org=is_member_of_different_org, + ) + + # Assertions + mock_normalize_domains.assert_called_once_with(mock_domain) + mock_get_requestor_email.assert_called_once_with(mock_requestor, [mock_domain]) + mock_validate_invitation.assert_called_once_with( + email, [mock_domain], mock_requestor, is_member_of_different_org + ) + self.assertEqual(str(context.exception), "Error sending email") + + @patch("registrar.utility.email_invitations.send_emails_to_domain_managers") + @patch("registrar.utility.email_invitations.validate_invitation") + @patch("registrar.utility.email_invitations.get_requestor_email") + @patch("registrar.utility.email_invitations.send_invitation_email") + @patch("registrar.utility.email_invitations.normalize_domains") + def test_send_domain_invitation_email_manager_emails_send_mail_exception( + self, + mock_normalize_domains, + mock_send_invitation_email, + mock_get_requestor_email, + mock_validate_invitation, + mock_send_domain_manager_emails, + ): + """Test sending domain invitation email for one domain and assert exception + when send_emails_to_domain_managers fails. + """ + # Setup + mock_domain = MagicMock(name="domain1") + mock_domain.name = "example.com" + mock_normalize_domains.return_value = [mock_domain] + + mock_requestor = MagicMock() + mock_requestor_email = "requestor@example.com" + mock_get_requestor_email.return_value = mock_requestor_email + + email = "invitee@example.com" + is_member_of_different_org = False + + mock_send_domain_manager_emails.side_effect = EmailSendingError("Error sending email") + + # Call and assert exception + with self.assertRaises(EmailSendingError) as context: + send_domain_invitation_email( + email=email, + requestor=mock_requestor, + domains=mock_domain, + is_member_of_different_org=is_member_of_different_org, + ) + + # Assertions + mock_normalize_domains.assert_called_once_with(mock_domain) + mock_get_requestor_email.assert_called_once_with(mock_requestor, [mock_domain]) + mock_validate_invitation.assert_called_once_with( + email, [mock_domain], mock_requestor, is_member_of_different_org + ) + mock_send_invitation_email.assert_called_once_with(email, mock_requestor_email, [mock_domain], None) + self.assertEqual(str(context.exception), "Error sending email") diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index ba660499e..c2bf22c30 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -1,8 +1,6 @@ from datetime import date from django.conf import settings -from registrar.models import DomainInvitation -from registrar.models.domain import Domain -from registrar.models.user_domain_role import UserDomainRole +from registrar.models import Domain, DomainInvitation, UserDomainRole from registrar.utility.errors import ( AlreadyDomainInvitedError, AlreadyDomainManagerError, @@ -73,7 +71,9 @@ def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, }, ) except EmailSendingError as err: - raise EmailSendingError(f"Could not send email manager notification to {user.email} for domains: {domain.name}") from err + raise EmailSendingError( + f"Could not send email manager notification to {user.email} for domains: {domain.name}" + ) from err def normalize_domains(domains): From 13c9e1c1135e3bb0b1fa936614721c214a14df69 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 16:35:04 -0500 Subject: [PATCH 165/201] lint --- src/registrar/tests/test_admin.py | 18 ++++++------------ src/registrar/tests/test_email_invitations.py | 8 ++++++++ src/registrar/tests/test_views_domain.py | 8 ++++++-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 673057e20..1decf02dd 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -165,19 +165,15 @@ class TestDomainInvitationAdmin(TestCase): response, "Domain invitations contain all individuals who have been invited to manage a .gov domain." ) self.assertContains(response, "Show more") - + @less_console_noise_decorator def test_has_change_form_description(self): """Tests if this model has a model description on the change form view""" self.client.force_login(self.superuser) - domain, _ = Domain.objects.get_or_create( - name="systemofadown.com" - ) + domain, _ = Domain.objects.get_or_create(name="systemofadown.com") - domain_invitation, _ = DomainInvitation.objects.get_or_create( - email="toxicity@systemofadown.com", domain=domain - ) + domain_invitation, _ = DomainInvitation.objects.get_or_create(email="toxicity@systemofadown.com", domain=domain) response = self.client.get( "/admin/registrar/domaininvitation/{}/change/".format(domain_invitation.pk), @@ -190,7 +186,7 @@ class TestDomainInvitationAdmin(TestCase): # Test for a description snippet self.assertContains( response, - "If you add someone to a domain here, it will trigger emails to the invitee and all managers of the domain when you click", + "If you add someone to a domain here, it will trigger emails to the invitee and all managers of the domain", ) @less_console_noise_decorator @@ -1983,15 +1979,13 @@ class TestUserDomainRoleAdmin(TestCase): response, "This table represents the managers who are assigned to each domain in the registrar" ) self.assertContains(response, "Show more") - + @less_console_noise_decorator def test_has_change_form_description(self): """Tests if this model has a model description on the change form view""" self.client.force_login(self.superuser) - domain, _ = Domain.objects.get_or_create( - name="systemofadown.com" - ) + domain, _ = Domain.objects.get_or_create(name="systemofadown.com") user_domain_role, _ = UserDomainRole.objects.get_or_create( user=self.superuser, domain=domain, role=[UserDomainRole.Roles.MANAGER] diff --git a/src/registrar/tests/test_email_invitations.py b/src/registrar/tests/test_email_invitations.py index 6a0423f4d..87384d3be 100644 --- a/src/registrar/tests/test_email_invitations.py +++ b/src/registrar/tests/test_email_invitations.py @@ -4,9 +4,12 @@ from datetime import date from registrar.utility.email import EmailSendingError from registrar.utility.email_invitations import send_domain_invitation_email +from api.tests.common import less_console_noise_decorator + class DomainInvitationEmail(unittest.TestCase): + @less_console_noise_decorator @patch("registrar.utility.email_invitations.send_templated_email") @patch("registrar.utility.email_invitations.UserDomainRole.objects.filter") @patch("registrar.utility.email_invitations.validate_invitation") @@ -71,6 +74,7 @@ class DomainInvitationEmail(unittest.TestCase): }, ) + @less_console_noise_decorator @patch("registrar.utility.email_invitations.send_templated_email") @patch("registrar.utility.email_invitations.UserDomainRole.objects.filter") @patch("registrar.utility.email_invitations.validate_invitation") @@ -170,6 +174,7 @@ class DomainInvitationEmail(unittest.TestCase): # Verify the total number of calls to send_templated_email self.assertEqual(mock_send_templated_email.call_count, 2) + @less_console_noise_decorator @patch("registrar.utility.email_invitations.validate_invitation") def test_send_domain_invitation_email_raises_invite_validation_exception(self, mock_validate_invitation): """Test sending domain invitation email for one domain and assert exception @@ -188,6 +193,7 @@ class DomainInvitationEmail(unittest.TestCase): self.assertEqual(str(context.exception), "Validation failed") mock_validate_invitation.assert_called_once() + @less_console_noise_decorator @patch("registrar.utility.email_invitations.get_requestor_email") def test_send_domain_invitation_email_raises_get_requestor_email_exception(self, mock_get_requestor_email): """Test sending domain invitation email for one domain and assert exception @@ -206,6 +212,7 @@ class DomainInvitationEmail(unittest.TestCase): self.assertEqual(str(context.exception), "Validation failed") mock_get_requestor_email.assert_called_once() + @less_console_noise_decorator @patch("registrar.utility.email_invitations.validate_invitation") @patch("registrar.utility.email_invitations.get_requestor_email") @patch("registrar.utility.email_invitations.send_invitation_email") @@ -254,6 +261,7 @@ class DomainInvitationEmail(unittest.TestCase): ) self.assertEqual(str(context.exception), "Error sending email") + @less_console_noise_decorator @patch("registrar.utility.email_invitations.send_emails_to_domain_managers") @patch("registrar.utility.email_invitations.validate_invitation") @patch("registrar.utility.email_invitations.get_requestor_email") diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index a9de8d6e7..45758e502 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -9,7 +9,7 @@ from registrar.utility.email import EmailSendingError from waffle.testutils import override_flag from api.tests.common import less_console_noise_decorator from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices -from .common import MockEppLib, MockSESClient, create_user # type: ignore +from .common import MockEppLib, create_user # type: ignore from django_webtest import WebTest # type: ignore import boto3_mocking # type: ignore @@ -766,7 +766,11 @@ class TestDomainManagers(TestDomainOverview): success_result = add_page.form.submit() mock_send_domain_email.assert_called_once_with( - email="mayor@igorville.gov", requestor=self.user, domains=self.domain, is_member_of_different_org=None, requested_user=user + email="mayor@igorville.gov", + requestor=self.user, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, ) self.assertEqual(success_result.status_code, 302) From 3420cc3329cf884ed3b31a9cb0adc53625a24cee Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 16:41:49 -0500 Subject: [PATCH 166/201] more linter --- src/registrar/utility/email_invitations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index c2bf22c30..3653d4290 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -76,7 +76,7 @@ def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, ) from err -def normalize_domains(domains): +def normalize_domains(domains: Domain | list[Domain]) -> list[Domain]: """Ensures domains is always a list.""" return [domains] if isinstance(domains, Domain) else domains From c0ce6561927b76b99ea6b154d8b9adbc74fdbf1d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:56:50 -0700 Subject: [PATCH 167/201] Account for duplicates --- .../commands/create_federal_portfolio.py | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 426a3ea23..0b2fd0887 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -375,15 +375,67 @@ class Command(BaseCommand): requests = DomainRequest.objects.filter( should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False ) - domains_dict = {domain.organization_name: domain for domain in domains} - requests_dict = {request.organization_name: request for request in requests} + + # Since domains / requests can share an org name, lets first create lists of duplicate names. + # If only one item exists in this list, we can just pull that info. + # If more than one exists, then we can determine what value we should choose. + domains_dict = {} + for domain in domains: + normalized_name = normalize_string(domain.organization_name) + domains_dict.setdefault(normalized_name, []).append(domain) + + requests_dict = {} + for request in requests: + normalized_name = normalize_string(request.organization_name) + requests_dict.setdefault(normalized_name, []).append(request) + suborgs_to_edit = Suborganization.objects.filter( Q(id__in=domains.values_list("sub_organization", flat=True)) | Q(id__in=requests.values_list("sub_organization", flat=True)) ) for suborg in suborgs_to_edit: - domain = domains_dict.get(suborg.name, None) - request = requests_dict.get(suborg.name, None) + normalized_suborg_name = normalize_string(suborg.name) + domains = domains_dict.get(normalized_suborg_name, []) + requests = requests_dict.get(normalized_suborg_name, []) + + domains_length = len(domains) + if domains_length == 0: + domain = None + elif domains_length == 1: + domain = domains[0] + else: + logger.info(f"in this loop (domains): {domains}") + reference = domains[0] + use_domain = all( + domain.city == reference.city and domain.state_territory == reference.state_territory + for domain in domains[1:] + ) + domain = reference if use_domain else None + + requests_length = len(requests) + if requests_length == 0: + request = None + elif requests_length == 1: + request = requests[0] + else: + logger.info(f"in this loop (requests): {requests}") + reference = requests[0] + use_domain = all( + ( + (request.city == reference.city and request.state_territory == reference.state_territory) + or ( + request.suborganization_city == reference.suborganization_city + and request.suborganization_state_territory == reference.suborganization_state_territory + ) + ) + for request in requests[1:] + ) + request = reference if use_domain else None + + if not domain and not request: + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + break # PRIORITY: # 1. Domain info @@ -414,3 +466,9 @@ class Command(BaseCommand): logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) + + def locations_match(item, reference): + return (item.city == reference.city and item.state_territory == reference.state_territory) or ( + item.suborganization_city == reference.suborganization_city + and item.suborganization_state_territory == reference.suborganization_state_territory + ) From 1456104df673bf25b031c578ee2e1a6ea6c99013 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:59:19 -0700 Subject: [PATCH 168/201] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 0b2fd0887..82d2e16a4 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -407,7 +407,10 @@ class Command(BaseCommand): logger.info(f"in this loop (domains): {domains}") reference = domains[0] use_domain = all( - domain.city == reference.city and domain.state_territory == reference.state_territory + domain.city + and domain.state_territory + and domain.city == reference.city + and domain.state_territory == reference.state_territory for domain in domains[1:] ) domain = reference if use_domain else None @@ -422,9 +425,16 @@ class Command(BaseCommand): reference = requests[0] use_domain = all( ( - (request.city == reference.city and request.state_territory == reference.state_territory) + ( + request.city + and request.state_territory + and request.city == reference.city + and request.state_territory == reference.state_territory + ) or ( - request.suborganization_city == reference.suborganization_city + request.suborganization_city + and request.suborganization_state_territory + and request.suborganization_city == reference.suborganization_city and request.suborganization_state_territory == reference.suborganization_state_territory ) ) From 06623c5f8e2e4f49deca02a176fc5299b7a9dfff Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:03:44 -0700 Subject: [PATCH 169/201] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 82d2e16a4..849e2cfaa 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -445,7 +445,7 @@ class Command(BaseCommand): if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - break + continue # PRIORITY: # 1. Domain info From 640fda873b40ff8baf2839a1c13061c114e3eae8 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:11:38 -0700 Subject: [PATCH 170/201] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 849e2cfaa..be0155dc5 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -398,49 +398,41 @@ class Command(BaseCommand): domains = domains_dict.get(normalized_suborg_name, []) requests = requests_dict.get(normalized_suborg_name, []) - domains_length = len(domains) - if domains_length == 0: - domain = None - elif domains_length == 1: - domain = domains[0] - else: - logger.info(f"in this loop (domains): {domains}") + # Try to get matching domain info + domain = None + if domains: reference = domains[0] - use_domain = all( - domain.city - and domain.state_territory - and domain.city == reference.city - and domain.state_territory == reference.state_territory - for domain in domains[1:] + locations_match = all( + d.city + and d.state_territory + and d.city == reference.city + and d.state_territory == reference.state_territory + for d in domains ) - domain = reference if use_domain else None + if locations_match: + domain = reference - requests_length = len(requests) - if requests_length == 0: - request = None - elif requests_length == 1: - request = requests[0] - else: - logger.info(f"in this loop (requests): {requests}") + # Try to get matching request info + request = None + if requests: reference = requests[0] - use_domain = all( + locations_match = all( ( - ( - request.city - and request.state_territory - and request.city == reference.city - and request.state_territory == reference.state_territory - ) - or ( - request.suborganization_city - and request.suborganization_state_territory - and request.suborganization_city == reference.suborganization_city - and request.suborganization_state_territory == reference.suborganization_state_territory - ) + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory ) - for request in requests[1:] + or ( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory + ) + for r in requests ) - request = reference if use_domain else None + if locations_match: + request = reference if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." From 4aa80abc8ce17aa1ba3c1d762fa8a959c1d4a727 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 15 Jan 2025 20:42:28 -0600 Subject: [PATCH 171/201] refactor to simplify and make duplication logic generic --- src/registrar/views/transfer_user.py | 227 +++++++++------------------ 1 file changed, 71 insertions(+), 156 deletions(-) diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index fdef91b43..37b6a3cc0 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -1,6 +1,6 @@ import logging -from django.db import transaction -from django.db.models import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel +from django.db import transaction, IntegrityError +from django.db.models import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel, Model, UniqueConstraint from django.shortcuts import render, get_object_or_404, redirect from django.views import View @@ -12,7 +12,8 @@ from django.contrib import messages from registrar.models.user_domain_role import UserDomainRole from registrar.models.user_portfolio_permission import UserPortfolioPermission -from typing import List + +from psycopg2 import errorcodes logger = logging.getLogger(__name__) @@ -65,9 +66,6 @@ class TransferUserView(View): with transaction.atomic(): change_logs = [] - self._delete_duplicate_user_domain_roles_and_log(selected_user, current_user, change_logs) - - self._delete_duplicate_user_portfolio_permissions_and_log(selected_user, current_user, change_logs) # Dynamically handle related fields self.transfer_related_fields_and_log(selected_user, current_user, change_logs) @@ -87,70 +85,6 @@ class TransferUserView(View): return redirect("admin:registrar_user_change", object_id=user_id) - def _delete_duplicate_user_portfolio_permissions_and_log(self, selected_user, current_user, change_logs): - """ - Check and remove duplicate UserPortfolioPermission objects from the selected_user based on portfolios associated with the current_user. - """ - try: - # Fetch portfolios associated with the current_user - current_user_portfolios = UserPortfolioPermission.objects.filter(user=current_user).values_list( - "portfolio_id", flat=True - ) - - # Identify duplicates in selected_user for these portfolios - duplicates = UserPortfolioPermission.objects.filter( - user=selected_user, portfolio_id__in=current_user_portfolios - ) - - duplicate_count = duplicates.count() - - if duplicate_count > 0: - # Log the specific duplicates before deletion for better traceability - duplicate_permissions = list(duplicates) - logger.debug(f"Duplicate permissions to be removed: {duplicate_permissions}") - - duplicates.delete() - logger.info( - f"Removed {duplicate_count} duplicate UserPortfolioPermission(s) from user_id {selected_user.id} for portfolios already associated with user_id {current_user.id}" - ) - change_logs.append( - f"Removed {duplicate_count} duplicate UserPortfolioPermission(s) from user_id {selected_user.id} for portfolios already associated with user_id {current_user.id}" - ) - - except Exception as e: - logger.error(f"Failed to check and remove duplicate UserPortfolioPermissions: {e}", exc_info=True) - raise - - def _delete_duplicate_user_domain_roles_and_log(self, selected_user, current_user, change_logs): - """ - Check and remove duplicate UserDomainRole objects from the selected_user based on domains associated with the current_user. - Retain one instance per domain to maintain data integrity. - """ - - try: - # Fetch domains associated with the current_user - current_user_domains = UserDomainRole.objects.filter(user=current_user).values_list("domain_id", flat=True) - - # Identify duplicates in selected_user for these domains - duplicates = UserDomainRole.objects.filter(user=selected_user, domain_id__in=current_user_domains) - - duplicate_count = duplicates.count() - - if duplicate_count > 0: - duplicates.delete() - logger.info( - f"Removed {duplicate_count} duplicate UserDomainRole(s) from user_id {selected_user.id} " - f"for domains already associated with user_id {current_user.id}" - ) - change_logs.append( - f"Removed {duplicate_count} duplicate UserDomainRole(s) from user_id {selected_user.id} " - f"for domains already associated with user_id {current_user.id}" - ) - - except Exception as e: - logger.error(f"Failed to check and remove duplicate UserDomainRoles: {e}", exc_info=True) - raise - def transfer_related_fields_and_log(self, selected_user, current_user, change_logs): """ Dynamically find all related fields to the User model and transfer them from selected_user to current_user. @@ -158,108 +92,89 @@ class TransferUserView(View): """ user_model = User - # Handle forward relationships for related_field in user_model._meta.get_fields(): - if related_field.is_relation and related_field.related_model: - if isinstance(related_field, ForeignKey): - self._handle_foreign_key(related_field, selected_user, current_user, change_logs) - elif isinstance(related_field, OneToOneField): + if related_field.is_relation: + # Field objects represent forward relationships + if isinstance(related_field, OneToOneField): self._handle_one_to_one(related_field, selected_user, current_user, change_logs) elif isinstance(related_field, ManyToManyField): self._handle_many_to_many(related_field, selected_user, current_user, change_logs) + elif isinstance(related_field, ForeignKey): + self._handle_foreign_key(related_field, selected_user, current_user, change_logs) + # Relationship objects represent reverse relationships elif isinstance(related_field, ManyToOneRel): - self._handle_many_to_one_rel(related_field, selected_user, current_user, change_logs) + # ManyToOneRel is a reverse ForeignKey + self._handle_foreign_key_reverse(related_field, selected_user, current_user, change_logs) + elif isinstance(related_field, OneToOneRel): + self._handle_one_to_one_reverse(related_field, selected_user, current_user, change_logs) + elif isinstance(related_field, ManyToManyRel): + self._handle_many_to_many_reverse(related_field, selected_user, current_user, change_logs) + else: + logger.error(f"Unknown relationship type for field {related_field}") + raise ValueError(f"Unknown relationship type for field {related_field}") - # # Handle reverse relationships - for related_object in user_model._meta.related_objects: - if isinstance(related_object, ManyToOneRel): - self._handle_many_to_one_rel(related_object, selected_user, current_user, change_logs) - elif isinstance(related_object.field, OneToOneField): - self._handle_one_to_one_reverse(related_object, selected_user, current_user, change_logs) - elif isinstance(related_object.field, ForeignKey): - self._handle_foreign_key_reverse(related_object, selected_user, current_user, change_logs) - elif isinstance(related_object.field, ManyToManyField): - self._handle_many_to_many_reverse(related_object, selected_user, current_user, change_logs) - - def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): - related_name = related_field.get_accessor_name() - related_manager = getattr(selected_user, related_name, None) - - if related_manager.count() > 0: - related_queryset = related_manager.all() - for obj in related_queryset: - setattr(obj, related_field.field.name, current_user) - obj.save() - self.log_change(selected_user, current_user, related_field.field.name, change_logs) - - def _handle_one_to_one(self, related_field: OneToOneField, selected_user, current_user, change_logs): - related_name = related_field.get_accessor_name() - related_object = getattr(selected_user, related_name, None) - - if related_object: - setattr(related_object, related_field.field.name, current_user) - related_object.save() + def _handle_foreign_key_reverse(self, related_field: ManyToOneRel, selected_user, current_user, change_logs): + # Handle reverse ForeignKey relationships + related_manager = getattr(selected_user, related_field.get_accessor_name(), None) + if related_manager and related_manager.exists(): + for related_object in related_manager.all(): + # use an atomic transaction to set a save point in case of a unique constraint violation + with transaction.atomic(): + try: + setattr(related_object, related_field.field.name, current_user) + related_object.save() + except IntegrityError as e: + if e.__cause__.pgcode == errorcodes.UNIQUE_VIOLATION: + # roll back to the savepoint, effectively ignoring this transaction + continue + else: + raise e self.log_change(selected_user, current_user, related_field.field.name, change_logs) - def _handle_many_to_many(self, related_field: ManyToManyField, selected_user, current_user, change_logs): - related_manager = getattr(selected_user, related_field.name, None) - if related_manager.count() > 0: - related_queryset = related_manager.all() - getattr(current_user, related_field.name).add(*related_queryset) + def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): + # Handle ForeignKey relationships + related_object = getattr(selected_user, related_field.name, None) + if related_object: + setattr(current_user, related_field.name, related_object) + current_user.save() self.log_change(selected_user, current_user, related_field.name, change_logs) - def _handle_many_to_one_rel( - self, related_object: ManyToOneRel, selected_user: User, current_user: User, change_logs: List[str] - ): - related_model = related_object.related_model - related_name = related_object.field.name + def _handle_one_to_one(self, related_field: OneToOneField, selected_user, current_user, change_logs): + # Handle OneToOne relationship + related_object = getattr(selected_user, related_field.name, None) + if related_object: + setattr(current_user, related_field.name, related_object) + current_user.save() + self.log_change(selected_user, current_user, related_field.name, change_logs) - related_queryset = related_model.objects.filter(**{related_name: selected_user}) + def _handle_many_to_many(self, related_field: ManyToManyField, selected_user, current_user, change_logs): + # Handle ManyToMany relationship + related_name = related_field.remote_field.name + related_manager = getattr(selected_user, related_name, None) + if related_manager and related_manager.exists(): + for instance in related_manager.all(): + getattr(instance, related_name).remove(selected_user) + getattr(instance, related_name).add(current_user) + self.log_change(selected_user, current_user, related_name, change_logs) - if related_queryset.count() > 0: - for obj in related_queryset: - setattr(obj, related_name, current_user) - obj.save() - self.log_change(selected_user, current_user, related_name, change_logs) + def _handle_many_to_many_reverse(self, related_field: ManyToManyRel, selected_user, current_user, change_logs): + # Handle reverse relationship + related_name = related_field.field.name + related_manager = getattr(selected_user, related_name, None) + if related_manager and related_manager.exists(): + for instance in related_manager.all(): + getattr(instance, related_name).remove(selected_user) + getattr(instance, related_name).add(current_user) + self.log_change(selected_user, current_user, related_name, change_logs) - def _handle_one_to_one_reverse( - self, related_object: OneToOneField, selected_user: User, current_user: User, change_logs: List[str] - ): - related_model = related_object.related_model - field_name = related_object.field.name - - try: - related_instance = related_model.objects.filter(**{field_name: selected_user}).first() + def _handle_one_to_one_reverse(self, related_field: OneToOneRel, selected_user, current_user, change_logs): + # Handle reverse relationship + field_name = related_field.get_accessor_name() + related_instance = getattr(selected_user, field_name, None) + if related_instance: setattr(related_instance, field_name, current_user) related_instance.save() self.log_change(selected_user, current_user, field_name, change_logs) - except related_model.DoesNotExist: - logger.warning(f"No related instance found for reverse OneToOneField {field_name} for {selected_user}") - - def _handle_foreign_key_reverse( - self, related_object: ForeignKey, selected_user: User, current_user: User, change_logs: List[str] - ): - related_model = related_object.related_model - field_name = related_object.field.name - - related_queryset = related_model.objects.filter(**{field_name: selected_user}) - - if related_queryset.count() > 0: - for obj in related_queryset: - setattr(obj, field_name, current_user) - obj.save() - self.log_change(selected_user, current_user, field_name, change_logs) - - def _handle_many_to_many_reverse( - self, related_object: ManyToManyField, selected_user: User, current_user: User, change_logs: List[str] - ): - related_model = related_object.related_model - field_name = related_object.field.name - - related_queryset = related_model.objects.filter(**{field_name: selected_user}) - if related_queryset.count() > 0: - getattr(current_user, field_name).add(*related_queryset) - self.log_change(selected_user, current_user, field_name, change_logs) @classmethod def log_change(cls, selected_user, current_user, field_name, change_logs): From 7b00c19c0e162f2a61189898bea3871e33c95a62 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 15 Jan 2025 21:23:30 -0600 Subject: [PATCH 172/201] refactor even more transfer user logic --- src/registrar/utility/db_helpers.py | 19 +++++++++++++ src/registrar/views/transfer_user.py | 41 ++++++++++++++-------------- 2 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 src/registrar/utility/db_helpers.py diff --git a/src/registrar/utility/db_helpers.py b/src/registrar/utility/db_helpers.py new file mode 100644 index 000000000..cb23d4f3e --- /dev/null +++ b/src/registrar/utility/db_helpers.py @@ -0,0 +1,19 @@ +from contextlib import contextmanager +from django.db import transaction, IntegrityError +from psycopg2 import errorcodes + +@contextmanager +def ignore_unique_violation(): + """ + Execute within an atomic transaction so that if a unique constraint violation occurs, + the individual transaction is rolled back without invalidating any larger transaction. + """ + with transaction.atomic(): + try: + yield + except IntegrityError as e: + if e.__cause__.pgcode == errorcodes.UNIQUE_VIOLATION: + # roll back to the savepoint, effectively ignoring this transaction + pass + else: + raise e diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index 37b6a3cc0..9dd959d69 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -15,6 +15,8 @@ from registrar.models.user_portfolio_permission import UserPortfolioPermission from psycopg2 import errorcodes +from registrar.utility.db_helpers import ignore_unique_violation + logger = logging.getLogger(__name__) @@ -118,33 +120,27 @@ class TransferUserView(View): related_manager = getattr(selected_user, related_field.get_accessor_name(), None) if related_manager and related_manager.exists(): for related_object in related_manager.all(): - # use an atomic transaction to set a save point in case of a unique constraint violation - with transaction.atomic(): - try: - setattr(related_object, related_field.field.name, current_user) - related_object.save() - except IntegrityError as e: - if e.__cause__.pgcode == errorcodes.UNIQUE_VIOLATION: - # roll back to the savepoint, effectively ignoring this transaction - continue - else: - raise e + with ignore_unique_violation(): + setattr(related_object, related_field.field.name, current_user) + related_object.save() self.log_change(selected_user, current_user, related_field.field.name, change_logs) def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): # Handle ForeignKey relationships related_object = getattr(selected_user, related_field.name, None) if related_object: - setattr(current_user, related_field.name, related_object) - current_user.save() + with ignore_unique_violation(): + setattr(current_user, related_field.name, related_object) + current_user.save() self.log_change(selected_user, current_user, related_field.name, change_logs) def _handle_one_to_one(self, related_field: OneToOneField, selected_user, current_user, change_logs): # Handle OneToOne relationship related_object = getattr(selected_user, related_field.name, None) if related_object: - setattr(current_user, related_field.name, related_object) - current_user.save() + with ignore_unique_violation(): + setattr(current_user, related_field.name, related_object) + current_user.save() self.log_change(selected_user, current_user, related_field.name, change_logs) def _handle_many_to_many(self, related_field: ManyToManyField, selected_user, current_user, change_logs): @@ -153,8 +149,9 @@ class TransferUserView(View): related_manager = getattr(selected_user, related_name, None) if related_manager and related_manager.exists(): for instance in related_manager.all(): - getattr(instance, related_name).remove(selected_user) - getattr(instance, related_name).add(current_user) + with ignore_unique_violation(): + getattr(instance, related_name).remove(selected_user) + getattr(instance, related_name).add(current_user) self.log_change(selected_user, current_user, related_name, change_logs) def _handle_many_to_many_reverse(self, related_field: ManyToManyRel, selected_user, current_user, change_logs): @@ -163,8 +160,9 @@ class TransferUserView(View): related_manager = getattr(selected_user, related_name, None) if related_manager and related_manager.exists(): for instance in related_manager.all(): - getattr(instance, related_name).remove(selected_user) - getattr(instance, related_name).add(current_user) + with ignore_unique_violation(): + getattr(instance, related_name).remove(selected_user) + getattr(instance, related_name).add(current_user) self.log_change(selected_user, current_user, related_name, change_logs) def _handle_one_to_one_reverse(self, related_field: OneToOneRel, selected_user, current_user, change_logs): @@ -172,8 +170,9 @@ class TransferUserView(View): field_name = related_field.get_accessor_name() related_instance = getattr(selected_user, field_name, None) if related_instance: - setattr(related_instance, field_name, current_user) - related_instance.save() + with ignore_unique_violation(): + setattr(related_instance, field_name, current_user) + related_instance.save() self.log_change(selected_user, current_user, field_name, change_logs) @classmethod From 931aff915fa0f8406fa6532575946c8fe4d8a396 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 15 Jan 2025 21:27:07 -0600 Subject: [PATCH 173/201] linters and remove debug logs --- src/registrar/utility/db_helpers.py | 1 + src/registrar/views/transfer_user.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/registrar/utility/db_helpers.py b/src/registrar/utility/db_helpers.py index cb23d4f3e..5b7e0392c 100644 --- a/src/registrar/utility/db_helpers.py +++ b/src/registrar/utility/db_helpers.py @@ -2,6 +2,7 @@ from contextlib import contextmanager from django.db import transaction, IntegrityError from psycopg2 import errorcodes + @contextmanager def ignore_unique_violation(): """ diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index 9dd959d69..31000257d 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -1,6 +1,6 @@ import logging -from django.db import transaction, IntegrityError -from django.db.models import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel, Model, UniqueConstraint +from django.db import transaction +from django.db.models import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel from django.shortcuts import render, get_object_or_404, redirect from django.views import View @@ -13,8 +13,6 @@ from django.contrib import messages from registrar.models.user_domain_role import UserDomainRole from registrar.models.user_portfolio_permission import UserPortfolioPermission -from psycopg2 import errorcodes - from registrar.utility.db_helpers import ignore_unique_violation logger = logging.getLogger(__name__) @@ -72,13 +70,10 @@ class TransferUserView(View): self.transfer_related_fields_and_log(selected_user, current_user, change_logs) # Success message if any related objects were updated - logger.debug(f"change_logs: {change_logs}") if change_logs: - logger.debug(f"change_logs: {change_logs}") success_message = f"Data transferred successfully for the following objects: {change_logs}" messages.success(request, success_message) - logger.debug("Deleting old user") selected_user.delete() messages.success(request, f"Deleted {selected_user} {selected_user.username}") except Exception as e: From 7e832c4a2bd01dc764c87ecb0143df446e2c1217 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Thu, 16 Jan 2025 09:34:53 -0600 Subject: [PATCH 174/201] remove unnecessary use of contextmanager --- src/registrar/views/transfer_user.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index 31000257d..fa66185ca 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -124,9 +124,8 @@ class TransferUserView(View): # Handle ForeignKey relationships related_object = getattr(selected_user, related_field.name, None) if related_object: - with ignore_unique_violation(): - setattr(current_user, related_field.name, related_object) - current_user.save() + setattr(current_user, related_field.name, related_object) + current_user.save() self.log_change(selected_user, current_user, related_field.name, change_logs) def _handle_one_to_one(self, related_field: OneToOneField, selected_user, current_user, change_logs): @@ -165,9 +164,8 @@ class TransferUserView(View): field_name = related_field.get_accessor_name() related_instance = getattr(selected_user, field_name, None) if related_instance: - with ignore_unique_violation(): - setattr(related_instance, field_name, current_user) - related_instance.save() + setattr(related_instance, field_name, current_user) + related_instance.save() self.log_change(selected_user, current_user, field_name, change_logs) @classmethod From a48215faec998275f26f695d3afc30742d97a278 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:23:11 -0700 Subject: [PATCH 175/201] Account for duplicate org names --- .../commands/create_federal_portfolio.py | 128 +++++++++--------- .../tests/test_management_scripts.py | 98 ++++++++++++++ 2 files changed, 162 insertions(+), 64 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index be0155dc5..48fee06ea 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -356,43 +356,56 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) def post_process_suborganization_fields(self, agencies): - """Post-process suborganization fields by pulling data from related domains and requests. - - This function updates suborganization city and state_territory fields based on - related domain information and domain request information. + """Updates suborganization city/state fields from domain and request data. + + Priority order for data: + 1. Domain information + 2. Domain request suborganization fields + 3. Domain request standard fields """ - # Assuming that org name, portfolio, and suborg all aren't null - # we assume that we want to add suborg info. - # as long as the org name doesnt match the portfolio name (as that implies it is the portfolio). - should_add_suborgs_filter = Q( - organization_name__isnull=False, + # Common filter between domaininformation / domain request. + # Filter by only the agencies we've updated thus far. + # Then, only process records without null portfolio, org name, or suborg name. + base_filter = Q( + federal_agency__in=agencies, portfolio__isnull=False, + organization_name__isnull=False, sub_organization__isnull=False, ) & ~Q(organization_name__iexact=F("portfolio__organization_name")) + + # First: Remove null city / state_territory values on domain info / domain requests. + # We want to add city data if there is data to add to begin with! domains = DomainInformation.objects.filter( - should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False + base_filter, + Q(city__isnull=False, state_territory__isnull=False), ) requests = DomainRequest.objects.filter( - should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False + base_filter, + ( + Q(city__isnull=False, state_territory__isnull=False) | + Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) + ), ) - # Since domains / requests can share an org name, lets first create lists of duplicate names. - # If only one item exists in this list, we can just pull that info. - # If more than one exists, then we can determine what value we should choose. + # Second: Group domains and requests by normalized organization name. + # This means that later down the line we have to account for "duplicate" org names. domains_dict = {} + requests_dict = {} for domain in domains: normalized_name = normalize_string(domain.organization_name) domains_dict.setdefault(normalized_name, []).append(domain) - requests_dict = {} for request in requests: normalized_name = normalize_string(request.organization_name) requests_dict.setdefault(normalized_name, []).append(request) + # Third: Get suborganizations to update suborgs_to_edit = Suborganization.objects.filter( Q(id__in=domains.values_list("sub_organization", flat=True)) | Q(id__in=requests.values_list("sub_organization", flat=True)) ) + + # Fourth: Process each suborg to add city / state territory info for suborg in suborgs_to_edit: normalized_suborg_name = normalize_string(suborg.name) domains = domains_dict.get(normalized_suborg_name, []) @@ -402,75 +415,62 @@ class Command(BaseCommand): domain = None if domains: reference = domains[0] - locations_match = all( - d.city - and d.state_territory - and d.city == reference.city + use_location_for_domain = all( + d.city == reference.city and d.state_territory == reference.state_territory for d in domains ) - if locations_match: + if use_location_for_domain: domain = reference # Try to get matching request info + # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". + # If not, take the safe route and just skip updating this particular record. request = None + use_suborg_location_for_request = True + use_location_for_request = True if requests: reference = requests[0] - locations_match = all( - ( - r.city - and r.state_territory - and r.city == reference.city - and r.state_territory == reference.state_territory - ) - or ( - r.suborganization_city - and r.suborganization_state_territory - and r.suborganization_city == reference.suborganization_city - and r.suborganization_state_territory == reference.suborganization_state_territory - ) + use_suborg_location_for_request = all( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory for r in requests ) - if locations_match: + use_location_for_request = all( + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory + for r in requests + ) + if use_suborg_location_for_request or use_location_for_request: request = reference if not domain and not request: - message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." - TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." + TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, message) continue # PRIORITY: # 1. Domain info # 2. Domain request requested suborg fields # 3. Domain request normal fields - city = None - if domain and domain.city: - city = normalize_string(domain.city, lowercase=False) - elif request and request.suborganization_city: - city = normalize_string(request.suborganization_city, lowercase=False) - elif request and request.city: - city = normalize_string(request.city, lowercase=False) + if domain: + suborg.city = normalize_string(domain.city, lowercase=False) + suborg.state_territory = domain.state_territory + elif request and use_suborg_location_for_request: + suborg.city = normalize_string(request.suborganization_city, lowercase=False) + suborg.state_territory = request.suborganization_state_territory + elif request and use_location_for_request: + suborg.city = normalize_string(request.city, lowercase=False) + suborg.state_territory = request.state_territory - state_territory = None - if domain and domain.state_territory: - state_territory = domain.state_territory - elif request and request.suborganization_state_territory: - state_territory = request.suborganization_state_territory - elif request and request.state_territory: - state_territory = request.state_territory - - if city: - suborg.city = city - - if suborg: - suborg.state_territory = state_territory - - logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") + logger.info( + f"Added city/state_territory to suborg: {suborg}. " + f"city - {suborg.city}, state - {suborg.state_territory}" + ) + # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) - - def locations_match(item, reference): - return (item.city == reference.city and item.state_territory == reference.state_territory) or ( - item.suborganization_city == reference.suborganization_city - and item.suborganization_state_territory == reference.suborganization_state_territory - ) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 8386088c8..769e9c805 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1920,6 +1920,104 @@ class TestCreateFederalPortfolio(TestCase): suborg_3 = Suborganization.objects.get(name=self.domain_info_3.organization_name) self.assertEqual(suborg_3.city, "Third City") self.assertEqual(suborg_3.state_territory, "FL") + + def test_post_process_suborganization_fields_multiple_locations(self): + """Test suborganization field updates when multiple domains/requests exist for the same org. + Tests that: + 1. Matching locations are accepted + 2. Non-matching locations are rejected + 3. Single location is always accepted + """ + # Test case 1: Multiple domains with matching locations + self.domain_info.organization_name = "matching" + self.domain_info.city = "Same City" + self.domain_info.state_territory = "NY" + self.domain_info.save() + + completed_domain_request( + name="matching" + ) + domain_info_copy = DomainInformation.objects.create( + organization_name="matching", + city="Same City", + state_territory="NY", + federal_agency=self.domain_info.federal_agency, + portfolio=self.domain_info.portfolio, + sub_organization=self.domain_info.sub_organization + ) + + # Test case 2: Multiple domains with non-matching locations + self.domain_info_2.organization_name = "non-matching" + self.domain_info_2.city = "City One" + self.domain_info_2.state_territory = "CA" + self.domain_info_2.save() + + domain_info_2_copy = DomainInformation.objects.create( + organization_name="non-matching", + city="City Two", # Different city + state_territory="CA", + federal_agency=self.domain_info_2.federal_agency, + portfolio=self.domain_info_2.portfolio, + sub_organization=self.domain_info_2.sub_organization + ) + + # Test case 3: Multiple requests with matching locations + self.domain_request.organization_name = "matching-requests" + self.domain_request.city = "Request City" + self.domain_request.state_territory = "TX" + self.domain_request.save() + + request_copy = DomainRequest.objects.create( + organization_name="matching-requests", + city="Request City", + state_territory="TX", + federal_agency=self.domain_request.federal_agency, + portfolio=self.domain_request.portfolio, + sub_organization=self.domain_request.sub_organization + ) + + # Test case 4: Multiple requests with non-matching locations + self.domain_request_2.organization_name = "non-matching-requests" + self.domain_request_2.city = "Request One" + self.domain_request_2.state_territory = "WA" + self.domain_request_2.save() + + request_2_copy = DomainRequest.objects.create( + organization_name="non-matching-requests", + city="Request Two", # Different city + state_territory="WA", + federal_agency=self.domain_request_2.federal_agency, + portfolio=self.domain_request_2.portfolio, + sub_organization=self.domain_request_2.sub_organization + ) + + # Run the portfolio creation + self.run_create_federal_portfolio( + agency_name="Test Federal Agency", + parse_requests=True, + parse_domains=True + ) + + # Verify results + # Case 1: Should use matching domain info + suborg_1 = Suborganization.objects.get(name="matching") + self.assertEqual(suborg_1.city, "Same City") + self.assertEqual(suborg_1.state_territory, "NY") + + # Case 2: Should not update due to mismatched locations + suborg_2 = Suborganization.objects.get(name="non-matching") + self.assertIsNone(suborg_2.city) + self.assertIsNone(suborg_2.state_territory) + + # Case 3: Should use matching request info + suborg_3 = Suborganization.objects.get(name="matching-requests") + self.assertEqual(suborg_3.city, "Request City") + self.assertEqual(suborg_3.state_territory, "TX") + + # Case 4: Should not update due to mismatched locations + suborg_4 = Suborganization.objects.get(name="non-matching-requests") + self.assertIsNone(suborg_4.city) + self.assertIsNone(suborg_4.state_territory) class TestPatchSuborganizations(MockDbForIndividualTests): From af7ce078bdb30993d73976511dc14d9d85182aee Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:56:04 -0700 Subject: [PATCH 176/201] Add unit tests --- .../commands/create_federal_portfolio.py | 21 +- .../tests/test_management_scripts.py | 223 +++++++++++------- 2 files changed, 152 insertions(+), 92 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 48fee06ea..534cf2c6f 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -357,17 +357,17 @@ class Command(BaseCommand): def post_process_suborganization_fields(self, agencies): """Updates suborganization city/state fields from domain and request data. - + Priority order for data: 1. Domain information - 2. Domain request suborganization fields + 2. Domain request suborganization fields 3. Domain request standard fields """ # Common filter between domaininformation / domain request. # Filter by only the agencies we've updated thus far. # Then, only process records without null portfolio, org name, or suborg name. base_filter = Q( - federal_agency__in=agencies, + federal_agency__in=agencies, portfolio__isnull=False, organization_name__isnull=False, sub_organization__isnull=False, @@ -382,9 +382,9 @@ class Command(BaseCommand): requests = DomainRequest.objects.filter( base_filter, ( - Q(city__isnull=False, state_territory__isnull=False) | - Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) - ), + Q(city__isnull=False, state_territory__isnull=False) + | Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) + ), ) # Second: Group domains and requests by normalized organization name. @@ -416,9 +416,7 @@ class Command(BaseCommand): if domains: reference = domains[0] use_location_for_domain = all( - d.city == reference.city - and d.state_territory == reference.state_territory - for d in domains + d.city == reference.city and d.state_territory == reference.state_territory for d in domains ) if use_location_for_domain: domain = reference @@ -450,7 +448,7 @@ class Command(BaseCommand): if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." - TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, message) + TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) continue # PRIORITY: @@ -467,10 +465,11 @@ class Command(BaseCommand): suborg.city = normalize_string(request.city, lowercase=False) suborg.state_territory = request.state_territory - logger.info( + message = ( f"Added city/state_territory to suborg: {suborg}. " f"city - {suborg.city}, state - {suborg.state_territory}" ) + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 769e9c805..655068493 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1845,6 +1845,7 @@ class TestCreateFederalPortfolio(TestCase): self.assertEqual(existing_portfolio.notes, "Old notes") self.assertEqual(existing_portfolio.creator, self.user) + @less_console_noise_decorator def test_post_process_suborganization_fields(self): """Test suborganization field updates from domain and request data. Also tests the priority order for updating city and state_territory: @@ -1920,104 +1921,164 @@ class TestCreateFederalPortfolio(TestCase): suborg_3 = Suborganization.objects.get(name=self.domain_info_3.organization_name) self.assertEqual(suborg_3.city, "Third City") self.assertEqual(suborg_3.state_territory, "FL") - - def test_post_process_suborganization_fields_multiple_locations(self): + + @less_console_noise_decorator + def test_post_process_suborganization_fields_duplicate_records(self): """Test suborganization field updates when multiple domains/requests exist for the same org. Tests that: - 1. Matching locations are accepted - 2. Non-matching locations are rejected - 3. Single location is always accepted + 1. City / state_territory us updated when all location info matches + 2. Updates are skipped when locations don't match + 3. Priority order is maintained across multiple records: + a. Domain information fields + b. Domain request suborganization fields + c. Domain request standard fields """ - # Test case 1: Multiple domains with matching locations - self.domain_info.organization_name = "matching" - self.domain_info.city = "Same City" - self.domain_info.state_territory = "NY" - self.domain_info.save() - - completed_domain_request( - name="matching" + # Case 1: Multiple records with all fields matching + matching_request_1 = completed_domain_request( + name="matching1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="matching org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) - domain_info_copy = DomainInformation.objects.create( - organization_name="matching", - city="Same City", - state_territory="NY", - federal_agency=self.domain_info.federal_agency, - portfolio=self.domain_info.portfolio, - sub_organization=self.domain_info.sub_organization + matching_request_1.approve() + domain_info_1 = DomainInformation.objects.get(domain_request=matching_request_1) + domain_info_1.city = "Domain Info City" + domain_info_1.state_territory = DomainRequest.StateTerritoryChoices.NEW_YORK + domain_info_1.save() + + matching_request_2 = completed_domain_request( + name="matching2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="matching org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + matching_request_2.approve() + domain_info_2 = DomainInformation.objects.get(domain_request=matching_request_2) + domain_info_2.city = "Domain Info City" + domain_info_2.state_territory = DomainRequest.StateTerritoryChoices.NEW_YORK + domain_info_2.save() - # Test case 2: Multiple domains with non-matching locations - self.domain_info_2.organization_name = "non-matching" - self.domain_info_2.city = "City One" - self.domain_info_2.state_territory = "CA" - self.domain_info_2.save() - - domain_info_2_copy = DomainInformation.objects.create( - organization_name="non-matching", - city="City Two", # Different city - state_territory="CA", - federal_agency=self.domain_info_2.federal_agency, - portfolio=self.domain_info_2.portfolio, - sub_organization=self.domain_info_2.sub_organization + # Case 2: Multiple records with only request fields (no domain info) + request_only_1 = completed_domain_request( + name="request1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="request org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + request_only_1.approve() + domain_info_3 = DomainInformation.objects.get(domain_request=request_only_1) + domain_info_3.city = None + domain_info_3.state_territory = None + domain_info_3.save() - # Test case 3: Multiple requests with matching locations - self.domain_request.organization_name = "matching-requests" - self.domain_request.city = "Request City" - self.domain_request.state_territory = "TX" - self.domain_request.save() - - request_copy = DomainRequest.objects.create( - organization_name="matching-requests", - city="Request City", - state_territory="TX", - federal_agency=self.domain_request.federal_agency, - portfolio=self.domain_request.portfolio, - sub_organization=self.domain_request.sub_organization + request_only_2 = completed_domain_request( + name="request2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="request org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + request_only_2.approve() + domain_info_4 = DomainInformation.objects.get(domain_request=request_only_2) + domain_info_4.city = None + domain_info_4.state_territory = None + domain_info_4.save() - # Test case 4: Multiple requests with non-matching locations - self.domain_request_2.organization_name = "non-matching-requests" - self.domain_request_2.city = "Request One" - self.domain_request_2.state_territory = "WA" - self.domain_request_2.save() - - request_2_copy = DomainRequest.objects.create( - organization_name="non-matching-requests", - city="Request Two", # Different city - state_territory="WA", - federal_agency=self.domain_request_2.federal_agency, - portfolio=self.domain_request_2.portfolio, - sub_organization=self.domain_request_2.sub_organization + # Case 3: Multiple records with only standard fields (no suborg) + standard_only_1 = completed_domain_request( + name="standard1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="standard org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + federal_agency=self.federal_agency, ) + standard_only_1.approve() + domain_info_5 = DomainInformation.objects.get(domain_request=standard_only_1) + domain_info_5.city = None + domain_info_5.state_territory = None + domain_info_5.save() - # Run the portfolio creation - self.run_create_federal_portfolio( - agency_name="Test Federal Agency", - parse_requests=True, - parse_domains=True + standard_only_2 = completed_domain_request( + name="standard2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="standard org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + federal_agency=self.federal_agency, ) + standard_only_2.approve() + domain_info_6 = DomainInformation.objects.get(domain_request=standard_only_2) + domain_info_6.city = None + domain_info_6.state_territory = None + domain_info_6.save() - # Verify results - # Case 1: Should use matching domain info - suborg_1 = Suborganization.objects.get(name="matching") - self.assertEqual(suborg_1.city, "Same City") - self.assertEqual(suborg_1.state_territory, "NY") + # Case 4: Multiple records with mismatched locations + mismatch_request_1 = completed_domain_request( + name="mismatch1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="mismatch org", + city="City One", + state_territory=DomainRequest.StateTerritoryChoices.FLORIDA, + federal_agency=self.federal_agency, + ) + mismatch_request_1.approve() + domain_info_5 = DomainInformation.objects.get(domain_request=mismatch_request_1) + domain_info_5.city = "Different City" + domain_info_5.state_territory = DomainRequest.StateTerritoryChoices.ALASKA + domain_info_5.save() - # Case 2: Should not update due to mismatched locations - suborg_2 = Suborganization.objects.get(name="non-matching") - self.assertIsNone(suborg_2.city) - self.assertIsNone(suborg_2.state_territory) + mismatch_request_2 = completed_domain_request( + name="mismatch2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="mismatch org", + city="City Two", + state_territory=DomainRequest.StateTerritoryChoices.HAWAII, + federal_agency=self.federal_agency, + ) + mismatch_request_2.approve() + domain_info_6 = DomainInformation.objects.get(domain_request=mismatch_request_2) + domain_info_6.city = "Another City" + domain_info_6.state_territory = DomainRequest.StateTerritoryChoices.CALIFORNIA + domain_info_6.save() - # Case 3: Should use matching request info - suborg_3 = Suborganization.objects.get(name="matching-requests") - self.assertEqual(suborg_3.city, "Request City") - self.assertEqual(suborg_3.state_territory, "TX") + # Run the portfolio creation script + self.run_create_federal_portfolio(agency_name="Test Federal Agency", parse_requests=True, parse_domains=True) - # Case 4: Should not update due to mismatched locations - suborg_4 = Suborganization.objects.get(name="non-matching-requests") - self.assertIsNone(suborg_4.city) - self.assertIsNone(suborg_4.state_territory) + # Case 1: Should use domain info values (highest priority) + matching_suborg = Suborganization.objects.get(name="matching org") + self.assertEqual(matching_suborg.city, "Domain Info City") + self.assertEqual(matching_suborg.state_territory, DomainRequest.StateTerritoryChoices.NEW_YORK) + + # Case 2: Should use suborg values (second priority) + request_suborg = Suborganization.objects.get(name="request org") + self.assertEqual(request_suborg.city, "Suborg City") + self.assertEqual(request_suborg.state_territory, DomainRequest.StateTerritoryChoices.CALIFORNIA) + + # Case 3: Should use standard values (lowest priority) + standard_suborg = Suborganization.objects.get(name="standard org") + self.assertEqual(standard_suborg.city, "Standard City") + self.assertEqual(standard_suborg.state_territory, DomainRequest.StateTerritoryChoices.TEXAS) + + # Case 4: Should skip update due to mismatched locations + mismatch_suborg = Suborganization.objects.get(name="mismatch org") + self.assertIsNone(mismatch_suborg.city) + self.assertIsNone(mismatch_suborg.state_territory) class TestPatchSuborganizations(MockDbForIndividualTests): From 46121542437201e75b9feb2ce529459fdbe3da20 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:05:41 -0700 Subject: [PATCH 177/201] lint part 2 --- .../commands/create_federal_portfolio.py | 153 ++++++++++-------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 534cf2c6f..c56b4ff6b 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -106,7 +106,7 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) # POST PROCESS STEP: Add additional suborg info where applicable. - updated_suborg_count = self.post_process_suborganization_fields(agencies) + updated_suborg_count = self.post_process_all_suborganization_fields(agencies) message = f"Added city and state_territory information to {updated_suborg_count} suborgs." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) @@ -355,10 +355,16 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - def post_process_suborganization_fields(self, agencies): - """Updates suborganization city/state fields from domain and request data. + def post_process_all_suborganization_fields(self, agencies): + """Batch updates suborganization locations from domain and request data. - Priority order for data: + Args: + agencies: List of FederalAgency objects to process + + Returns: + int: Number of suborganizations updated + + Priority for location data: 1. Domain information 2. Domain request suborganization fields 3. Domain request standard fields @@ -407,69 +413,82 @@ class Command(BaseCommand): # Fourth: Process each suborg to add city / state territory info for suborg in suborgs_to_edit: - normalized_suborg_name = normalize_string(suborg.name) - domains = domains_dict.get(normalized_suborg_name, []) - requests = requests_dict.get(normalized_suborg_name, []) - - # Try to get matching domain info - domain = None - if domains: - reference = domains[0] - use_location_for_domain = all( - d.city == reference.city and d.state_territory == reference.state_territory for d in domains - ) - if use_location_for_domain: - domain = reference - - # Try to get matching request info - # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". - # If not, take the safe route and just skip updating this particular record. - request = None - use_suborg_location_for_request = True - use_location_for_request = True - if requests: - reference = requests[0] - use_suborg_location_for_request = all( - r.suborganization_city - and r.suborganization_state_territory - and r.suborganization_city == reference.suborganization_city - and r.suborganization_state_territory == reference.suborganization_state_territory - for r in requests - ) - use_location_for_request = all( - r.city - and r.state_territory - and r.city == reference.city - and r.state_territory == reference.state_territory - for r in requests - ) - if use_suborg_location_for_request or use_location_for_request: - request = reference - - if not domain and not request: - message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." - TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) - continue - - # PRIORITY: - # 1. Domain info - # 2. Domain request requested suborg fields - # 3. Domain request normal fields - if domain: - suborg.city = normalize_string(domain.city, lowercase=False) - suborg.state_territory = domain.state_territory - elif request and use_suborg_location_for_request: - suborg.city = normalize_string(request.suborganization_city, lowercase=False) - suborg.state_territory = request.suborganization_state_territory - elif request and use_location_for_request: - suborg.city = normalize_string(request.city, lowercase=False) - suborg.state_territory = request.state_territory - - message = ( - f"Added city/state_territory to suborg: {suborg}. " - f"city - {suborg.city}, state - {suborg.state_territory}" - ) - TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + self.post_process_suborganization_fields(suborg, domains_dict, requests_dict) # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) + + def post_process_suborganization_fields(self, suborg, domains_dict, requests_dict): + """Updates a single suborganization's location data if valid. + + Args: + suborg: Suborganization to update + domains_dict: Dict of domain info records grouped by org name + requests_dict: Dict of domain requests grouped by org name + + Priority matches parent method. Updates are skipped if location data conflicts + between multiple records of the same type. + """ + normalized_suborg_name = normalize_string(suborg.name) + domains = domains_dict.get(normalized_suborg_name, []) + requests = requests_dict.get(normalized_suborg_name, []) + + # Try to get matching domain info + domain = None + if domains: + reference = domains[0] + use_location_for_domain = all( + d.city == reference.city and d.state_territory == reference.state_territory for d in domains + ) + if use_location_for_domain: + domain = reference + + # Try to get matching request info + # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". + # If not, take the safe route and just skip updating this particular record. + request = None + use_suborg_location_for_request = True + use_location_for_request = True + if requests: + reference = requests[0] + use_suborg_location_for_request = all( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory + for r in requests + ) + use_location_for_request = all( + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory + for r in requests + ) + if use_suborg_location_for_request or use_location_for_request: + request = reference + + if not domain and not request: + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." + TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) + return + + # PRIORITY: + # 1. Domain info + # 2. Domain request requested suborg fields + # 3. Domain request normal fields + if domain: + suborg.city = normalize_string(domain.city, lowercase=False) + suborg.state_territory = domain.state_territory + elif request and use_suborg_location_for_request: + suborg.city = normalize_string(request.suborganization_city, lowercase=False) + suborg.state_territory = request.suborganization_state_territory + elif request and use_location_for_request: + suborg.city = normalize_string(request.city, lowercase=False) + suborg.state_territory = request.state_territory + + message = ( + f"Added city/state_territory to suborg: {suborg}. " + f"city - {suborg.city}, state - {suborg.state_territory}" + ) + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) From 6a49f9e373a6db7f99a50f402cb4938ab6c7597a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Jan 2025 14:13:51 -0500 Subject: [PATCH 178/201] requesting entity working - code still wip --- src/registrar/assets/js/uswds-edited.js | 71 +++------ src/registrar/assets/src/js/getgov/main.js | 2 +- src/registrar/forms/domain_request_wizard.py | 150 ++++++++++++++++--- src/registrar/views/domain_request.py | 10 +- 4 files changed, 163 insertions(+), 70 deletions(-) diff --git a/src/registrar/assets/js/uswds-edited.js b/src/registrar/assets/js/uswds-edited.js index 60502050f..9e3922c9c 100644 --- a/src/registrar/assets/js/uswds-edited.js +++ b/src/registrar/assets/js/uswds-edited.js @@ -1038,7 +1038,7 @@ const noop = () => {}; * @param {string} value The new value of the element */ const changeElementValue = function (el) { - let value = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : ""; + let value = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : ""; const elementToChange = el; elementToChange.value = value; const event = new CustomEvent("change", { @@ -1168,22 +1168,14 @@ const enhanceComboBox = _comboBoxEl => { placeholder }); } - // DOTGOV - allowing for defaultValue to be empty - //if (defaultValue) { - // for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - // const optionEl = selectEl.options[i]; - // if (optionEl.value === defaultValue) { - // selectedOption = optionEl; - // break; - // } - // } - //} - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if ((optionEl.value === defaultValue) || (!optionEl.value && !defaultValue)) { - selectedOption = optionEl; - break; - } + if (defaultValue) { + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if (optionEl.value === defaultValue) { + selectedOption = optionEl; + break; + } + } } /** @@ -1234,11 +1226,9 @@ const enhanceComboBox = _comboBoxEl => { input.setAttribute(key, value); })); comboBoxEl.insertAdjacentElement("beforeend", input); - // DOTGOV - modified the aria-label of the clear input button to Reset selection to reflect changed button behavior - // comboBoxEl.insertAdjacentHTML("beforeend", Sanitizer.escapeHTML` - +   @@ -1374,12 +1364,8 @@ const displayList = el => { for (let i = 0, len = selectEl.options.length; i < len; i += 1) { const optionEl = selectEl.options[i]; const optionId = `${listOptionBaseId}${options.length}`; - // DOTGOV: modified combobox to allow for options with blank value - //if (optionEl.value && (disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { - if ((disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { - // DOTGOV: modified combobox to allow blank option value selections to be considered selected - //if (selectEl.value && optionEl.value === selectEl.value) { - if (selectEl.value && optionEl.value === selectEl.value || (!selectEl.value && !optionEl.value)) { + if (optionEl.value && (disableFiltering || isPristine || !inputValue || regex.test(optionEl.text))) { + if (selectEl.value && optionEl.value === selectEl.value) { selectedItemId = optionId; } if (disableFiltering && !firstFoundId && regex.test(optionEl.text)) { @@ -1514,28 +1500,17 @@ const resetSelection = el => { } = getComboBoxContext(el); const selectValue = selectEl.value; const inputValue = (inputEl.value || "").toLowerCase(); - // DOTGOV - allow for option value to be empty string - //if (selectValue) { - // for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - // const optionEl = selectEl.options[i]; - // if (optionEl.value === selectValue) { - // if (inputValue !== optionEl.text) { - // changeElementValue(inputEl, optionEl.text); - // } - // comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); - // return; - // } - // } - //} - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if ((!selectValue && !optionEl.value) || optionEl.value === selectValue) { - if (inputValue !== optionEl.text) { - changeElementValue(inputEl, optionEl.text); - } - comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); - return; - } + if (selectValue) { + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if (optionEl.value === selectValue) { + if (inputValue !== optionEl.text) { + changeElementValue(inputEl, optionEl.text); + } + comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); + return; + } + } } if (inputValue) { changeElementValue(inputEl); diff --git a/src/registrar/assets/src/js/getgov/main.js b/src/registrar/assets/src/js/getgov/main.js index 6ff402aa4..9f3156cf7 100644 --- a/src/registrar/assets/src/js/getgov/main.js +++ b/src/registrar/assets/src/js/getgov/main.js @@ -31,7 +31,7 @@ initializeUrbanizationToggle(); userProfileListener(); finishUserSetupListener(); -loadInitialValuesForComboBoxes(); +//loadInitialValuesForComboBoxes(); handleRequestingEntityFieldset(); diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 636a41760..e40355d61 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -77,6 +77,20 @@ class RequestingEntityForm(RegistrarForm): (obj.id, str(obj)) for obj in queryset ] + [("other", "Other (enter your suborganization manually)")] + @classmethod + def from_database(cls, obj: DomainRequest | None): + """Returns a dict of form field values gotten from `obj`. + Overrides RegistrarForm method in order to set sub_organization to 'other' + on GETs of the RequestingEntityForm.""" + if obj is None: + return {} + # get the domain request as a dict, per usual method + domain_request_dict = {name: getattr(obj, name) for name in cls.declared_fields.keys()} # type: ignore + # set sub_organization to 'other' if is_requesting_new_suborganization is True + if obj.is_requesting_new_suborganization(): + domain_request_dict["sub_organization"] = "other" + return domain_request_dict + def clean_sub_organization(self): """On suborganization clean, set the suborganization value to None if the user is requesting a custom suborganization (as it doesn't exist yet)""" @@ -102,42 +116,132 @@ class RequestingEntityForm(RegistrarForm): ) return name + # def full_clean(self): + # """Validation logic to remove the custom suborganization value before clean is triggered. + # Without this override, the form will throw an 'invalid option' error.""" + # # Remove the custom other field before cleaning + # data = self.data.copy() if self.data else None + + # # Remove the 'other' value from suborganization if it exists. + # # This is a special value that tracks if the user is requesting a new suborg. + # suborganization = self.data.get("portfolio_requesting_entity-sub_organization") + # if suborganization and "other" in suborganization: + # data["portfolio_requesting_entity-sub_organization"] = "" + + # # Set the modified data back to the form + # self.data = data + + # # Call the parent's full_clean method + # super().full_clean() + def full_clean(self): - """Validation logic to remove the custom suborganization value before clean is triggered. + """Validation logic to temporarily remove the custom suborganization value before clean is triggered. Without this override, the form will throw an 'invalid option' error.""" - # Remove the custom other field before cleaning - data = self.data.copy() if self.data else None + logger.debug("full_clean") + # Ensure self.data is not None before proceeding + if self.data: + # handle case where form has been submitted + logger.debug("form was submitted") + # Create a copy of the data for manipulation + data = self.data.copy() - # Remove the 'other' value from suborganization if it exists. - # This is a special value that tracks if the user is requesting a new suborg. - suborganization = self.data.get("portfolio_requesting_entity-sub_organization") - if suborganization and "other" in suborganization: - data["portfolio_requesting_entity-sub_organization"] = "" + # Retrieve sub_organization + suborganization = data.get("portfolio_requesting_entity-sub_organization") - # Set the modified data back to the form - self.data = data + logger.debug(f"suborganization submitted as {suborganization}") + # # Determine if "other" should be stored in _original_suborganization + # if not suborganization: + # logger.debug("suborg stored as other") + # self._original_suborganization = "other" + # else: + self._original_suborganization = suborganization + + # If the original value was "other", clear it for validation + if self._original_suborganization == "other": + data["portfolio_requesting_entity-sub_organization"] = "" + + # Set the modified data back to the form + self.data = data + else: + # handle case of a GET + suborganization = None + if self.initial and "sub_organization" in self.initial: + print("suborg in self.initial") + suborganization = self.initial["sub_organization"] + print(self.initial["sub_organization"]) + print(suborganization) + # Check if is_requesting_new_suborganization is True + is_requesting_new_suborganization = False + if self.initial and "is_requesting_new_suborganization" in self.initial: + # Call the method if it exists + print(self.initial["is_requesting_new_suborganization"]()) + is_requesting_new_suborganization = self.initial["is_requesting_new_suborganization"]() + + # Determine if "other" should be set + if is_requesting_new_suborganization and suborganization is None: + print("presetting to other") + self._original_suborganization = "other" + else: + self._original_suborganization = suborganization + print("self.data does not exist") + print(self.initial) + # # Handle the initial GET request case + # self._original_suborganization = None # Call the parent's full_clean method super().full_clean() + # Restore "other" if there are errors + if self.errors: + logger.debug(f"errors detected: {self.errors}; resetting original_sub_organization") + self.data["portfolio_requesting_entity-sub_organization"] = self._original_suborganization + + + # def clean(self): + # """Custom clean implementation to handle our desired logic flow for suborganization. + # Given that these fields often rely on eachother, we need to do this in the parent function.""" + # cleaned_data = super().clean() + + # # Do some custom error validation if the requesting entity is a suborg. + # # Otherwise, just validate as normal. + # suborganization = self.cleaned_data.get("sub_organization") + # is_requesting_new_suborganization = self.cleaned_data.get("is_requesting_new_suborganization") + + # # Get the value of the yes/no checkbox from RequestingEntityYesNoForm. + # # Since self.data stores this as a string, we need to convert "True" => True. + # requesting_entity_is_suborganization = self.data.get( + # "portfolio_requesting_entity-requesting_entity_is_suborganization" + # ) + # if requesting_entity_is_suborganization == "True": + # if is_requesting_new_suborganization: + # # Validate custom suborganization fields + # if not cleaned_data.get("requested_suborganization") and "requested_suborganization" not in self.errors: + # self.add_error("requested_suborganization", "Enter the name of your suborganization.") + # if not cleaned_data.get("suborganization_city"): + # self.add_error("suborganization_city", "Enter the city where your suborganization is located.") + # if not cleaned_data.get("suborganization_state_territory"): + # self.add_error( + # "suborganization_state_territory", + # "Select the state, territory, or military post where your suborganization is located.", + # ) + # elif not suborganization: + # self.add_error("sub_organization", "Suborganization is required.") + + # return cleaned_data + def clean(self): - """Custom clean implementation to handle our desired logic flow for suborganization. - Given that these fields often rely on eachother, we need to do this in the parent function.""" + """Custom clean implementation to handle our desired logic flow for suborganization.""" cleaned_data = super().clean() - # Do some custom error validation if the requesting entity is a suborg. - # Otherwise, just validate as normal. - suborganization = self.cleaned_data.get("sub_organization") - is_requesting_new_suborganization = self.cleaned_data.get("is_requesting_new_suborganization") - - # Get the value of the yes/no checkbox from RequestingEntityYesNoForm. - # Since self.data stores this as a string, we need to convert "True" => True. + # Get the cleaned data + suborganization = cleaned_data.get("sub_organization") + is_requesting_new_suborganization = cleaned_data.get("is_requesting_new_suborganization") requesting_entity_is_suborganization = self.data.get( "portfolio_requesting_entity-requesting_entity_is_suborganization" ) + if requesting_entity_is_suborganization == "True": if is_requesting_new_suborganization: - # Validate custom suborganization fields if not cleaned_data.get("requested_suborganization") and "requested_suborganization" not in self.errors: self.add_error("requested_suborganization", "Enter the name of your suborganization.") if not cleaned_data.get("suborganization_city"): @@ -150,6 +254,12 @@ class RequestingEntityForm(RegistrarForm): elif not suborganization: self.add_error("sub_organization", "Suborganization is required.") + # If there are errors, restore the "other" value for rendering + if self.errors and getattr(self, "_original_suborganization", None) == "other": + self.cleaned_data["sub_organization"] = self._original_suborganization + elif not self.data and getattr(self, "_original_suborganization", None) == "other": + self.cleaned_data["sub_organization"] = self._original_suborganization + return cleaned_data diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index 9754b0d0c..e0225aab3 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -368,7 +368,7 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView): and from the database if `use_db` is True (provided that record exists). An empty form will be provided if neither of those are true. """ - + logger.debug(f"get_forms({step},{use_post},{use_db},{files})") kwargs = { "files": files, "prefix": self.steps.current, @@ -385,6 +385,7 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView): for form in forms: data = form.from_database(self.domain_request) if self.has_pk() else None + logger.debug(data) if use_post: instantiated.append(form(self.request.POST, **kwargs)) elif use_db: @@ -562,6 +563,13 @@ class RequestingEntity(DomainRequestWizard): template_name = "domain_request_requesting_entity.html" forms = [forms.RequestingEntityYesNoForm, forms.RequestingEntityForm] + #for debugging: + def get(self, request, *args, **kwargs): + """This method handles GET requests.""" + logger.debug("in get") + + return super().get(request, *args, **kwargs) + def save(self, forms: list): """Override of save to clear or associate certain suborganization data depending on what the user wishes to do. For instance, we want to add a suborganization From 088140a37d6dd96f3e98fa896fd5f15b02c31b1e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Jan 2025 14:24:43 -0500 Subject: [PATCH 179/201] clean up of requesting entity --- src/registrar/forms/domain_request_wizard.py | 72 +------------------- src/registrar/views/domain_request.py | 9 --- 2 files changed, 2 insertions(+), 79 deletions(-) diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index e40355d61..e452c01b1 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -116,44 +116,18 @@ class RequestingEntityForm(RegistrarForm): ) return name - # def full_clean(self): - # """Validation logic to remove the custom suborganization value before clean is triggered. - # Without this override, the form will throw an 'invalid option' error.""" - # # Remove the custom other field before cleaning - # data = self.data.copy() if self.data else None - - # # Remove the 'other' value from suborganization if it exists. - # # This is a special value that tracks if the user is requesting a new suborg. - # suborganization = self.data.get("portfolio_requesting_entity-sub_organization") - # if suborganization and "other" in suborganization: - # data["portfolio_requesting_entity-sub_organization"] = "" - - # # Set the modified data back to the form - # self.data = data - - # # Call the parent's full_clean method - # super().full_clean() def full_clean(self): """Validation logic to temporarily remove the custom suborganization value before clean is triggered. Without this override, the form will throw an 'invalid option' error.""" - logger.debug("full_clean") # Ensure self.data is not None before proceeding if self.data: # handle case where form has been submitted - logger.debug("form was submitted") # Create a copy of the data for manipulation data = self.data.copy() - # Retrieve sub_organization + # Retrieve sub_organization and store in _original_suborganization suborganization = data.get("portfolio_requesting_entity-sub_organization") - - logger.debug(f"suborganization submitted as {suborganization}") - # # Determine if "other" should be stored in _original_suborganization - # if not suborganization: - # logger.debug("suborg stored as other") - # self._original_suborganization = "other" - # else: self._original_suborganization = suborganization # If the original value was "other", clear it for validation @@ -166,69 +140,27 @@ class RequestingEntityForm(RegistrarForm): # handle case of a GET suborganization = None if self.initial and "sub_organization" in self.initial: - print("suborg in self.initial") suborganization = self.initial["sub_organization"] - print(self.initial["sub_organization"]) - print(suborganization) + # Check if is_requesting_new_suborganization is True is_requesting_new_suborganization = False if self.initial and "is_requesting_new_suborganization" in self.initial: # Call the method if it exists - print(self.initial["is_requesting_new_suborganization"]()) is_requesting_new_suborganization = self.initial["is_requesting_new_suborganization"]() # Determine if "other" should be set if is_requesting_new_suborganization and suborganization is None: - print("presetting to other") self._original_suborganization = "other" else: self._original_suborganization = suborganization - print("self.data does not exist") - print(self.initial) - # # Handle the initial GET request case - # self._original_suborganization = None # Call the parent's full_clean method super().full_clean() # Restore "other" if there are errors if self.errors: - logger.debug(f"errors detected: {self.errors}; resetting original_sub_organization") self.data["portfolio_requesting_entity-sub_organization"] = self._original_suborganization - - # def clean(self): - # """Custom clean implementation to handle our desired logic flow for suborganization. - # Given that these fields often rely on eachother, we need to do this in the parent function.""" - # cleaned_data = super().clean() - - # # Do some custom error validation if the requesting entity is a suborg. - # # Otherwise, just validate as normal. - # suborganization = self.cleaned_data.get("sub_organization") - # is_requesting_new_suborganization = self.cleaned_data.get("is_requesting_new_suborganization") - - # # Get the value of the yes/no checkbox from RequestingEntityYesNoForm. - # # Since self.data stores this as a string, we need to convert "True" => True. - # requesting_entity_is_suborganization = self.data.get( - # "portfolio_requesting_entity-requesting_entity_is_suborganization" - # ) - # if requesting_entity_is_suborganization == "True": - # if is_requesting_new_suborganization: - # # Validate custom suborganization fields - # if not cleaned_data.get("requested_suborganization") and "requested_suborganization" not in self.errors: - # self.add_error("requested_suborganization", "Enter the name of your suborganization.") - # if not cleaned_data.get("suborganization_city"): - # self.add_error("suborganization_city", "Enter the city where your suborganization is located.") - # if not cleaned_data.get("suborganization_state_territory"): - # self.add_error( - # "suborganization_state_territory", - # "Select the state, territory, or military post where your suborganization is located.", - # ) - # elif not suborganization: - # self.add_error("sub_organization", "Suborganization is required.") - - # return cleaned_data - def clean(self): """Custom clean implementation to handle our desired logic flow for suborganization.""" cleaned_data = super().clean() diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index e0225aab3..5fccfc2f2 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -368,7 +368,6 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView): and from the database if `use_db` is True (provided that record exists). An empty form will be provided if neither of those are true. """ - logger.debug(f"get_forms({step},{use_post},{use_db},{files})") kwargs = { "files": files, "prefix": self.steps.current, @@ -385,7 +384,6 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView): for form in forms: data = form.from_database(self.domain_request) if self.has_pk() else None - logger.debug(data) if use_post: instantiated.append(form(self.request.POST, **kwargs)) elif use_db: @@ -562,13 +560,6 @@ class PortfolioDomainRequestWizard(DomainRequestWizard): class RequestingEntity(DomainRequestWizard): template_name = "domain_request_requesting_entity.html" forms = [forms.RequestingEntityYesNoForm, forms.RequestingEntityForm] - - #for debugging: - def get(self, request, *args, **kwargs): - """This method handles GET requests.""" - logger.debug("in get") - - return super().get(request, *args, **kwargs) def save(self, forms: list): """Override of save to clear or associate certain suborganization data From 4649198867a48408be1ca09452ecbdc62988811a Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:53:16 -0700 Subject: [PATCH 180/201] Update domain.py --- src/registrar/models/domain.py | 52 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 5e186efff..a388196ca 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1665,28 +1665,6 @@ class Domain(TimeStampedModel, DomainHelper): ) return err.code - def _fetch_contacts(self, contact_data): - """Fetch contact info.""" - choices = PublicContact.ContactTypeChoices - # We expect that all these fields get populated, - # so we can create these early, rather than waiting. - contacts_dict = { - choices.ADMINISTRATIVE: None, - choices.SECURITY: None, - choices.TECHNICAL: None, - } - for domainContact in contact_data: - req = commands.InfoContact(id=domainContact.contact) - data = registry.send(req, cleaned=True).res_data[0] - - # Map the object we recieved from EPP to a PublicContact - mapped_object = self.map_epp_contact_to_public_contact(data, domainContact.contact, domainContact.type) - - # Find/create it in the DB - in_db = self._get_or_create_public_contact(mapped_object) - contacts_dict[in_db.contact_type] = in_db.registry_id - return contacts_dict - def _get_or_create_contact(self, contact: PublicContact): """Try to fetch info about a contact. Create it if it does not exist.""" logger.info("_get_or_create_contact() -> Fetching contact info") @@ -1869,8 +1847,8 @@ class Domain(TimeStampedModel, DomainHelper): missingSecurity = True missingTech = True - if len(cleaned.get("_contacts")) < 3: - for contact in cleaned.get("_contacts"): + if len(cleaned.get("contacts")) < 3: + for contact in cleaned.get("contacts"): if contact.type == PublicContact.ContactTypeChoices.ADMINISTRATIVE: missingAdmin = False if contact.type == PublicContact.ContactTypeChoices.SECURITY: @@ -1890,7 +1868,7 @@ class Domain(TimeStampedModel, DomainHelper): technical_contact.save() logger.info( - "_add_missing_contacts_if_unknown => " + "_add_missing_contacts_if_unknown => In function. Values are " f"missingAdmin: {missingAdmin}, missingSecurity: {missingSecurity}, missingTech: {missingTech}" ) @@ -2070,6 +2048,30 @@ class Domain(TimeStampedModel, DomainHelper): if contacts and isinstance(contacts, list) and len(contacts) > 0: cleaned_contacts = self._fetch_contacts(contacts) return cleaned_contacts + + def _fetch_contacts(self, contact_data): + """Fetch contact info.""" + choices = PublicContact.ContactTypeChoices + # We expect that all these fields get populated, + # so we can create these early, rather than waiting. + contacts_dict = { + choices.ADMINISTRATIVE: None, + choices.SECURITY: None, + choices.TECHNICAL: None, + } + for domainContact in contact_data: + req = commands.InfoContact(id=domainContact.contact) + data = registry.send(req, cleaned=True).res_data[0] + logger.info(f"_fetch_contacts => this is the data: {data}") + + # Map the object we recieved from EPP to a PublicContact + mapped_object = self.map_epp_contact_to_public_contact(data, domainContact.contact, domainContact.type) + logger.info(f"_fetch_contacts => mapped_object: {mapped_object}") + + # Find/create it in the DB + in_db = self._get_or_create_public_contact(mapped_object) + contacts_dict[in_db.contact_type] = in_db.registry_id + return contacts_dict def _get_hosts(self, hosts): cleaned_hosts = [] From 511e9610e27b014f7a505f9c2f6c4c441823d615 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:55:50 -0700 Subject: [PATCH 181/201] Update domain.py --- src/registrar/models/domain.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index a388196ca..ad549879b 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1847,8 +1847,10 @@ class Domain(TimeStampedModel, DomainHelper): missingSecurity = True missingTech = True - if len(cleaned.get("contacts")) < 3: - for contact in cleaned.get("contacts"): + # Potential collision - mismatch between _contacts and contacts? + # But the ID wouldnt match in this case because the default is being grabbed? + if len(cleaned.get("_contacts")) < 3: + for contact in cleaned.get("_contacts"): if contact.type == PublicContact.ContactTypeChoices.ADMINISTRATIVE: missingAdmin = False if contact.type == PublicContact.ContactTypeChoices.SECURITY: From f9fa8772e585fc30d25d439532b3a1ea8b0a5ed5 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Jan 2025 15:09:32 -0500 Subject: [PATCH 182/201] cleanup and lint --- src/registrar/assets/js/uswds-edited.js | 36 ++++----- .../assets/src/js/getgov/combobox.js | 77 ------------------- src/registrar/assets/src/js/getgov/main.js | 3 - src/registrar/forms/domain_request_wizard.py | 15 ++-- src/registrar/views/domain_request.py | 2 +- 5 files changed, 27 insertions(+), 106 deletions(-) delete mode 100644 src/registrar/assets/src/js/getgov/combobox.js diff --git a/src/registrar/assets/js/uswds-edited.js b/src/registrar/assets/js/uswds-edited.js index 9e3922c9c..9d4dd2e51 100644 --- a/src/registrar/assets/js/uswds-edited.js +++ b/src/registrar/assets/js/uswds-edited.js @@ -29,8 +29,6 @@ * - tooltip dynamic content updated to include nested element (for better sizing control) * - modal exposed to window to be accessible in other js files * - fixed bug in createHeaderButton which added newlines to header button tooltips - * - modified combobox to allow for blank values in list - * - modified aria label for X button in combobox to reflect modified behavior of button * - modified combobox to handle error class */ @@ -1169,13 +1167,13 @@ const enhanceComboBox = _comboBoxEl => { }); } if (defaultValue) { - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if (optionEl.value === defaultValue) { - selectedOption = optionEl; - break; - } - } + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if (optionEl.value === defaultValue) { + selectedOption = optionEl; + break; + } + } } /** @@ -1501,16 +1499,16 @@ const resetSelection = el => { const selectValue = selectEl.value; const inputValue = (inputEl.value || "").toLowerCase(); if (selectValue) { - for (let i = 0, len = selectEl.options.length; i < len; i += 1) { - const optionEl = selectEl.options[i]; - if (optionEl.value === selectValue) { - if (inputValue !== optionEl.text) { - changeElementValue(inputEl, optionEl.text); - } - comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); - return; - } - } + for (let i = 0, len = selectEl.options.length; i < len; i += 1) { + const optionEl = selectEl.options[i]; + if (optionEl.value === selectValue) { + if (inputValue !== optionEl.text) { + changeElementValue(inputEl, optionEl.text); + } + comboBoxEl.classList.add(COMBO_BOX_PRISTINE_CLASS); + return; + } + } } if (inputValue) { changeElementValue(inputEl); diff --git a/src/registrar/assets/src/js/getgov/combobox.js b/src/registrar/assets/src/js/getgov/combobox.js deleted file mode 100644 index e0ecc92ad..000000000 --- a/src/registrar/assets/src/js/getgov/combobox.js +++ /dev/null @@ -1,77 +0,0 @@ -import { hideElement, showElement } from './helpers.js'; - -export function loadInitialValuesForComboBoxes() { - var overrideDefaultClearButton = true; - var isTyping = false; - - document.addEventListener('DOMContentLoaded', (event) => { - handleAllComboBoxElements(); - }); - - function handleAllComboBoxElements() { - const comboBoxElements = document.querySelectorAll(".usa-combo-box"); - comboBoxElements.forEach(comboBox => { - const input = comboBox.querySelector("input"); - const select = comboBox.querySelector("select"); - if (!input || !select) { - console.warn("No combobox element found"); - return; - } - // Set the initial value of the combobox - let initialValue = select.getAttribute("data-default-value"); - let clearInputButton = comboBox.querySelector(".usa-combo-box__clear-input"); - if (!clearInputButton) { - console.warn("No clear element found"); - return; - } - - // Override the default clear button behavior such that it no longer clears the input, - // it just resets to the data-initial-value. - // Due to the nature of how uswds works, this is slightly hacky. - - // Input event listener to detect typing - input.addEventListener("input", () => { - isTyping = true; - }); - - // Blur event listener to reset typing state - input.addEventListener("blur", () => { - isTyping = false; - }); - - // Hide the reset button when there is nothing to reset. - // Do this once on init, then everytime a change occurs. - updateClearButtonVisibility(select, initialValue, clearInputButton) - select.addEventListener("change", () => { - updateClearButtonVisibility(select, initialValue, clearInputButton) - }); - - // Change the default input behaviour - have it reset to the data default instead - clearInputButton.addEventListener("click", (e) => { - if (overrideDefaultClearButton) { - e.preventDefault(); - e.stopPropagation(); - input.click(); - // Find the dropdown option with the desired value - const dropdownOptions = document.querySelectorAll(".usa-combo-box__list-option"); - if (dropdownOptions) { - dropdownOptions.forEach(option => { - if (option.getAttribute("data-value") === initialValue) { - // Simulate a click event on the dropdown option - option.click(); - } - }); - } - } - }); - }); - } - - function updateClearButtonVisibility(select, initialValue, clearInputButton) { - if (select.value === initialValue) { - hideElement(clearInputButton); - }else { - showElement(clearInputButton) - } - } -} diff --git a/src/registrar/assets/src/js/getgov/main.js b/src/registrar/assets/src/js/getgov/main.js index 9f3156cf7..a077da929 100644 --- a/src/registrar/assets/src/js/getgov/main.js +++ b/src/registrar/assets/src/js/getgov/main.js @@ -3,7 +3,6 @@ import { initDomainValidators } from './domain-validators.js'; import { initFormsetsForms, triggerModalOnDsDataForm, nameserversFormListener } from './formset-forms.js'; import { initializeUrbanizationToggle } from './urbanization.js'; import { userProfileListener, finishUserSetupListener } from './user-profile.js'; -import { loadInitialValuesForComboBoxes } from './combobox.js'; import { handleRequestingEntityFieldset } from './requesting-entity.js'; import { initDomainsTable } from './table-domains.js'; import { initDomainRequestsTable } from './table-domain-requests.js'; @@ -31,8 +30,6 @@ initializeUrbanizationToggle(); userProfileListener(); finishUserSetupListener(); -//loadInitialValuesForComboBoxes(); - handleRequestingEntityFieldset(); initDomainsTable(); diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index e452c01b1..12d3b74bf 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -73,12 +73,14 @@ class RequestingEntityForm(RegistrarForm): self.fields["sub_organization"].queryset = queryset # Modify the choices to include "other" so that form can display options properly - self.fields["sub_organization"].choices = [("", "--Select--")] + [ - (obj.id, str(obj)) for obj in queryset - ] + [("other", "Other (enter your suborganization manually)")] + self.fields["sub_organization"].choices = ( + [("", "--Select--")] + + [(obj.id, str(obj)) for obj in queryset] + + [("other", "Other (enter your suborganization manually)")] + ) @classmethod - def from_database(cls, obj: DomainRequest | None): + def from_database(cls, obj: DomainRequest | Contact | None): """Returns a dict of form field values gotten from `obj`. Overrides RegistrarForm method in order to set sub_organization to 'other' on GETs of the RequestingEntityForm.""" @@ -86,9 +88,11 @@ class RequestingEntityForm(RegistrarForm): return {} # get the domain request as a dict, per usual method domain_request_dict = {name: getattr(obj, name) for name in cls.declared_fields.keys()} # type: ignore + # set sub_organization to 'other' if is_requesting_new_suborganization is True - if obj.is_requesting_new_suborganization(): + if isinstance(obj, DomainRequest) and obj.is_requesting_new_suborganization(): domain_request_dict["sub_organization"] = "other" + return domain_request_dict def clean_sub_organization(self): @@ -116,7 +120,6 @@ class RequestingEntityForm(RegistrarForm): ) return name - def full_clean(self): """Validation logic to temporarily remove the custom suborganization value before clean is triggered. Without this override, the form will throw an 'invalid option' error.""" diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index 5fccfc2f2..bff3e5c00 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -560,7 +560,7 @@ class PortfolioDomainRequestWizard(DomainRequestWizard): class RequestingEntity(DomainRequestWizard): template_name = "domain_request_requesting_entity.html" forms = [forms.RequestingEntityYesNoForm, forms.RequestingEntityForm] - + def save(self, forms: list): """Override of save to clear or associate certain suborganization data depending on what the user wishes to do. For instance, we want to add a suborganization From 1ece895ac6a6c51d34f41b03a3210ea428185d1b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:15:36 -0700 Subject: [PATCH 183/201] Add test for race condition --- src/registrar/models/domain.py | 29 ++++++++------ src/registrar/tests/test_models_domain.py | 48 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index ad549879b..94ab21bde 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1939,6 +1939,8 @@ class Domain(TimeStampedModel, DomainHelper): Additionally, capture and cache old hosts and contacts from cache if they don't exist in cleaned """ + + # object reference issue between self._cache vs cleaned? old_cache_hosts = self._cache.get("hosts") old_cache_contacts = self._cache.get("contacts") @@ -2111,19 +2113,20 @@ class Domain(TimeStampedModel, DomainHelper): # Save to DB if it doesn't exist already. if db_contact.count() == 0: # Doesn't run custom save logic, just saves to DB - try: - public_contact.save(skip_epp_save=True) - logger.info(f"Created a new PublicContact: {public_contact}") - except IntegrityError as err: - logger.error( - "_get_or_create_public_contact() => tried to create a duplicate public contact: " - f"{err}", exc_info=True - ) - return PublicContact.objects.get( - registry_id=public_contact.registry_id, - contact_type=public_contact.contact_type, - domain=self, - ) + public_contact.save(skip_epp_save=True) + # try: + # public_contact.save(skip_epp_save=True) + # logger.info(f"Created a new PublicContact: {public_contact}") + # except IntegrityError as err: + # logger.error( + # "_get_or_create_public_contact() => tried to create a duplicate public contact: " + # f"{err}", exc_info=True + # ) + # return PublicContact.objects.get( + # registry_id=public_contact.registry_id, + # contact_type=public_contact.contact_type, + # domain=self, + # ) # Append the item we just created return public_contact diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 15a88a608..56d5cf2be 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -348,6 +348,54 @@ class TestDomainCache(MockEppLib): class TestDomainCreation(MockEppLib): """Rule: An approved domain request must result in a domain""" + def test_get_security_email_during_unknown_state_race_condition(self): + """ + Scenario: A domain is accessed for the first time + Given a domain in UNKNOWN state with a security contact in registry + When get_security_email is called during state transition + Then the security contact is fetched from registry + And only one security contact exists in database + And the security email matches the registry contact + And no duplicate contact creation is attempted + """ + domain, _ = Domain.objects.get_or_create(name="defaultsecurity.gov") + + # Store original method + original_filter = PublicContact.objects.filter + + def mock_filter(*args, **kwargs): + # First call returns empty queryset to simulate contact not existing + result = original_filter(*args, **kwargs) + if kwargs.get('contact_type') == PublicContact.ContactTypeChoices.SECURITY: + # Create the duplicate contact after the check but before the save + duplicate = PublicContact( + domain=domain, + contact_type=PublicContact.ContactTypeChoices.SECURITY, + registry_id="defaultSec", + email="dotgov@cisa.dhs.gov", + name="Registry Customer Service" + ) + duplicate.save(skip_epp_save=True) + return result + + with patch.object(PublicContact.objects, 'filter', side_effect=mock_filter): + try: + security_email = domain.get_security_email() + except IntegrityError: + self.fail( + "IntegrityError was raised during contact creation due to a race condition. " + "This indicates that concurrent contact creation is not working in some cases. " + "The error occurs when two processes try to create the same contact simultaneously. " + "Expected behavior: gracefully handle duplicate creation and return existing contact." + ) + + # Verify only one contact exists + security_contacts = PublicContact.objects.filter( + domain=domain, + contact_type=PublicContact.ContactTypeChoices.SECURITY + ) + self.assertEqual(security_contacts.count(), 1) + self.assertEqual(security_email, "dotgov@cisa.dhs.gov") @boto3_mocking.patching def test_approved_domain_request_creates_domain_locally(self): From 4de586005e0ed88583d343dd989d1b1bc1a09a63 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:40:03 -0700 Subject: [PATCH 184/201] Test part 2 --- src/registrar/models/domain.py | 32 +++++++++++-------- src/registrar/tests/test_models_domain.py | 39 +++++++++++++++-------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 94ab21bde..aaafd6aa8 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -2086,11 +2086,13 @@ class Domain(TimeStampedModel, DomainHelper): def _get_or_create_public_contact(self, public_contact: PublicContact): """Tries to find a PublicContact object in our DB. If it can't, it'll create it. Returns PublicContact""" + logger.info(f"in function") db_contact = PublicContact.objects.filter( registry_id=public_contact.registry_id, contact_type=public_contact.contact_type, domain=self, ) + logger.info(f"db_contact {db_contact}") # If we find duplicates, log it and delete the oldest ones. if db_contact.count() > 1: @@ -2113,20 +2115,22 @@ class Domain(TimeStampedModel, DomainHelper): # Save to DB if it doesn't exist already. if db_contact.count() == 0: # Doesn't run custom save logic, just saves to DB - public_contact.save(skip_epp_save=True) - # try: - # public_contact.save(skip_epp_save=True) - # logger.info(f"Created a new PublicContact: {public_contact}") - # except IntegrityError as err: - # logger.error( - # "_get_or_create_public_contact() => tried to create a duplicate public contact: " - # f"{err}", exc_info=True - # ) - # return PublicContact.objects.get( - # registry_id=public_contact.registry_id, - # contact_type=public_contact.contact_type, - # domain=self, - # ) + try: + public_contact.save(skip_epp_save=True) + logger.info(f"Created a new PublicContact: {public_contact}") + # In rare cases, _add_missing_contacts_if_unknown will cause a race condition with this function. + # This is because it calls .save(), which is called here. + # + except IntegrityError as err: + logger.error( + "_get_or_create_public_contact() => tried to create a duplicate public contact: " + f"{err}", exc_info=True + ) + return PublicContact.objects.get( + registry_id=public_contact.registry_id, + contact_type=public_contact.contact_type, + domain=self, + ) # Append the item we just created return public_contact diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 56d5cf2be..bb1974464 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -348,26 +348,30 @@ class TestDomainCache(MockEppLib): class TestDomainCreation(MockEppLib): """Rule: An approved domain request must result in a domain""" - def test_get_security_email_during_unknown_state_race_condition(self): + + def test_get_security_email_race_condition(self): """ - Scenario: A domain is accessed for the first time - Given a domain in UNKNOWN state with a security contact in registry - When get_security_email is called during state transition - Then the security contact is fetched from registry + Scenario: Two processes try to create the same security contact simultaneously + Given a domain in UNKNOWN state + When a race condition occurs during contact creation + Then no IntegrityError is raised And only one security contact exists in database - And the security email matches the registry contact - And no duplicate contact creation is attempted + And the correct security email is returned """ domain, _ = Domain.objects.get_or_create(name="defaultsecurity.gov") # Store original method original_filter = PublicContact.objects.filter - + self.first_call = True def mock_filter(*args, **kwargs): - # First call returns empty queryset to simulate contact not existing + """ Simulates the race condition by creating a + duplicate contact between filter check and save + """ result = original_filter(*args, **kwargs) - if kwargs.get('contact_type') == PublicContact.ContactTypeChoices.SECURITY: - # Create the duplicate contact after the check but before the save + + # Return empty queryset for first call. Otherwise just proceed as normal. + if self.first_call: + self.first_call = False duplicate = PublicContact( domain=domain, contact_type=PublicContact.ContactTypeChoices.SECURITY, @@ -376,11 +380,20 @@ class TestDomainCreation(MockEppLib): name="Registry Customer Service" ) duplicate.save(skip_epp_save=True) + return PublicContact.objects.none() + return result with patch.object(PublicContact.objects, 'filter', side_effect=mock_filter): try: - security_email = domain.get_security_email() + public_contact = PublicContact( + domain=domain, + contact_type=PublicContact.ContactTypeChoices.SECURITY, + registry_id="defaultSec", + email="dotgov@cisa.dhs.gov", + name="Registry Customer Service" + ) + returned_public_contact = domain._get_or_create_public_contact(public_contact) except IntegrityError: self.fail( "IntegrityError was raised during contact creation due to a race condition. " @@ -395,7 +408,7 @@ class TestDomainCreation(MockEppLib): contact_type=PublicContact.ContactTypeChoices.SECURITY ) self.assertEqual(security_contacts.count(), 1) - self.assertEqual(security_email, "dotgov@cisa.dhs.gov") + self.assertEqual(returned_public_contact, security_contacts.get()) @boto3_mocking.patching def test_approved_domain_request_creates_domain_locally(self): From cfcdd8826cb799701207ebab68e8aa06f11fe889 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:04:10 -0700 Subject: [PATCH 185/201] Squash bug --- src/registrar/models/domain.py | 5 +++-- src/registrar/tests/test_models_domain.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index aaafd6aa8..f7e76cbc4 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -2116,8 +2116,9 @@ class Domain(TimeStampedModel, DomainHelper): if db_contact.count() == 0: # Doesn't run custom save logic, just saves to DB try: - public_contact.save(skip_epp_save=True) - logger.info(f"Created a new PublicContact: {public_contact}") + with transaction.atomic(): + public_contact.save(skip_epp_save=True) + logger.info(f"Created a new PublicContact: {public_contact}") # In rare cases, _add_missing_contacts_if_unknown will cause a race condition with this function. # This is because it calls .save(), which is called here. # diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index bb1974464..8fde89fd0 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -349,6 +349,7 @@ class TestDomainCache(MockEppLib): class TestDomainCreation(MockEppLib): """Rule: An approved domain request must result in a domain""" + @less_console_noise_decorator def test_get_security_email_race_condition(self): """ Scenario: Two processes try to create the same security contact simultaneously From 2ad8e0268adebcf32b0d5092374a0c012f8c18d6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Jan 2025 19:05:37 -0500 Subject: [PATCH 186/201] changes to domain request data full report and test --- src/registrar/tests/common.py | 8 ++++++ src/registrar/tests/test_reports.py | 25 +++++++++++------- src/registrar/utility/csv_export.py | 39 +++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index c0083068d..2eff53d39 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -578,6 +578,10 @@ class MockDb(TestCase): creator=cls.custom_superuser, federal_agency=cls.federal_agency_3, organization_type="federal" ) + cls.suborganization_1, _ = Suborganization.objects.get_or_create( + name="SubOrg 1", portfolio=cls.portfolio_1, city="Nashville", state_territory="TN", + ) + current_date = get_time_aware_date(datetime(2024, 4, 2)) # Create start and end dates using timedelta @@ -848,6 +852,7 @@ class MockDb(TestCase): status=DomainRequest.DomainRequestStatus.IN_REVIEW, name="city2.gov", portfolio=cls.portfolio_1, + sub_organization=cls.suborganization_1, ) cls.domain_request_3 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, @@ -863,6 +868,9 @@ class MockDb(TestCase): cls.domain_request_5 = completed_domain_request( status=DomainRequest.DomainRequestStatus.APPROVED, name="city5.gov", + requested_suborganization="requested_suborg", + suborganization_city="SanFran", + suborganization_state_territory="CA", ) cls.domain_request_6 = completed_domain_request( status=DomainRequest.DomainRequestStatus.STARTED, diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index b11500ea9..cabaa048d 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -729,6 +729,7 @@ class ExportDataTest(MockDbForIndividualTests, MockEppLib): # "Submitted at", "Status", "Domain type", + "Portfolio", "Federal type", "Federal agency", "Organization name", @@ -736,6 +737,10 @@ class ExportDataTest(MockDbForIndividualTests, MockEppLib): "City", "State/territory", "Region", + "Suborganization", + "Requested suborg", + "Suborg city", + "Suborg state/territory", "Creator first name", "Creator last name", "Creator email", @@ -765,27 +770,29 @@ class ExportDataTest(MockDbForIndividualTests, MockEppLib): expected_content = ( # Header - "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," - "City,State/territory,Region,Creator first name,Creator last name,Creator email," + "Domain request,Status,Domain type,Portfolio,Federal type,Federal agency,Organization name," + "Election office,City,State/territory,Region,Suborganization,Requested suborg,Suborg city," + "Suborg state/territory,Creator first name,Creator last name,Creator email," "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,NY,2,,,,0,1,city1.gov,,,,," - "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,NY,2,,,,0,1," + "city5.gov,Approved,Federal,No,Executive,,Testorg,N/A,,NY,2,requested_suborg,SanFran,CA,,,,,1,0," + "city1.gov,Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,There is more," + "Testy Tester testy2@town.com,,city.com,\n" + "city2.gov,In review,Federal,Yes,Executive,Portfolio 1 Federal Agency,,N/A,,,2,SubOrg 1,,,,,,,0," + "1,city1.gov,,,,,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city3.gov,Submitted,Federal,Yes,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,,,,,0,1," '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' 'Testy Tester testy2@town.com",' 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," + "city4.gov,Submitted,City,No,Executive,,Testorg,Yes,,NY,2,,,,,,,,0,1,city1.gov,Testy," "Tester,testy@town.com," "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," "Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,NY,2,,,,0,1,city1.gov,,,,," + "city6.gov,Submitted,Federal,Yes,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,,,,,0,1,city1.gov,,,,," "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 6e5773ebb..1bb53a7a3 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1660,6 +1660,27 @@ class DomainRequestExport(BaseExport): default=F("organization_name"), output_field=CharField(), ), + "converted_city": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__city")), + # Otherwise, return the natively assigned value + default=F("city"), + output_field=CharField(), + ), + "converted_state_territory": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("state_territory"), + output_field=CharField(), + ), + "converted_suborganization_name": Case( + # When sub_organization is present, use its name + When(sub_organization__isnull=False, then=F("sub_organization__name")), + # Otherwise, return empty string + default=Value(""), + output_field=CharField(), + ), "converted_so_email": Case( # When portfolio is present, use its value instead When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), @@ -1786,6 +1807,10 @@ class DomainRequestExport(BaseExport): status = model.get("status") status_display = DomainRequest.DomainRequestStatus.get_status_label(status) if status else None + # Handle the portfolio field. Display as a Yes/No + portfolio = model.get("portfolio") + portfolio_display = "Yes" if portfolio is not None else "No" + # Handle the region field. state_territory = model.get("state_territory") region = get_region(state_territory) if state_territory else None @@ -1819,6 +1844,7 @@ class DomainRequestExport(BaseExport): "Election office": human_readable_election_board, "Federal type": human_readable_federal_type, "Domain type": human_readable_org_type, + "Portfolio": portfolio_display, "Request additional details": additional_details, # Annotated fields - passed into the request dict. "Creator approved domains count": model.get("creator_approved_domains_count", 0), @@ -1827,6 +1853,10 @@ class DomainRequestExport(BaseExport): "Other contacts": model.get("all_other_contacts"), "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. + "Suborganization": model.get("converted_suborganization_name"), + "Requested suborg": model.get("requested_suborganization"), + "Suborg city": model.get("suborganization_city"), + "Suborg state/territory": model.get("suborganization_state_territory"), "Federal agency": model.get("converted_federal_agency"), "SO first name": model.get("converted_senior_official_first_name"), "SO last name": model.get("converted_senior_official_last_name"), @@ -1838,8 +1868,8 @@ class DomainRequestExport(BaseExport): "Investigator": model.get("investigator__email"), # Untouched fields "Organization name": model.get("converted_organization_name"), - "City": model.get("city"), - "State/territory": model.get("state_territory"), + "City": model.get("converted_city"), + "State/territory": model.get("converted_state_territory"), "Request purpose": model.get("purpose"), "CISA regional representative": model.get("cisa_representative_email"), "Last submitted date": model.get("last_submitted_date"), @@ -2006,6 +2036,7 @@ class DomainRequestDataFull(DomainRequestExport): "Last status update", "Status", "Domain type", + "Portfolio", "Federal type", "Federal agency", "Organization name", @@ -2013,6 +2044,10 @@ class DomainRequestDataFull(DomainRequestExport): "City", "State/territory", "Region", + "Suborganization", + "Requested suborg", + "Suborg city", + "Suborg state/territory", "Creator first name", "Creator last name", "Creator email", From 75665ff378c5216fc7281f0637c24db8cb956cf2 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Jan 2025 19:31:11 -0500 Subject: [PATCH 187/201] fixed other tests and linted --- src/registrar/tests/common.py | 5 ++++- src/registrar/tests/test_management_scripts.py | 10 +++++++++- src/registrar/tests/test_reports.py | 4 ++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 2eff53d39..bb65ef6b1 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -579,7 +579,10 @@ class MockDb(TestCase): ) cls.suborganization_1, _ = Suborganization.objects.get_or_create( - name="SubOrg 1", portfolio=cls.portfolio_1, city="Nashville", state_territory="TN", + name="SubOrg 1", + portfolio=cls.portfolio_1, + city="Nashville", + state_territory="TN", ) current_date = get_time_aware_date(datetime(2024, 4, 2)) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 655068493..536d1e760 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -2101,6 +2101,10 @@ class TestPatchSuborganizations(MockDbForIndividualTests): 1. Fewest spaces 2. Most leading capitals """ + # Delete any other suborganizations defined in the initial test dataset + DomainRequest.objects.all().delete() + Suborganization.objects.all().delete() + Suborganization.objects.create(name="Test Organization ", portfolio=self.portfolio_1) Suborganization.objects.create(name="test organization", portfolio=self.portfolio_1) Suborganization.objects.create(name="Test Organization", portfolio=self.portfolio_1) @@ -2114,6 +2118,10 @@ class TestPatchSuborganizations(MockDbForIndividualTests): @less_console_noise_decorator def test_hardcoded_record(self): """Tests that our hardcoded records update as we expect them to""" + # Delete any other suborganizations defined in the initial test dataset + DomainRequest.objects.all().delete() + Suborganization.objects.all().delete() + # Create orgs with old and new name formats old_name = "USDA/OC" new_name = "USDA, Office of Communications" @@ -2123,7 +2131,7 @@ class TestPatchSuborganizations(MockDbForIndividualTests): self.run_patch_suborganizations() - # Verify only the new one remains + # Verify only the new one of the two remains self.assertEqual(Suborganization.objects.count(), 1) remaining = Suborganization.objects.first() self.assertEqual(remaining.name, new_name) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index cabaa048d..9d410e430 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -792,8 +792,8 @@ class ExportDataTest(MockDbForIndividualTests, MockEppLib): "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," "Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Yes,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,,,,,0,1,city1.gov,,,,," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," + "city6.gov,Submitted,Federal,Yes,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,,,,,0,1,city1.gov," + ",,,,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) From a0d6e1ec3356e9765e855b9c5a0766c734cacad8 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Jan 2025 10:06:40 -0700 Subject: [PATCH 188/201] cleanup --- src/registrar/models/domain.py | 24 +++++-------- src/registrar/tests/test_models_domain.py | 42 ++++++++++++----------- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index f7e76cbc4..2ace5f22d 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1847,10 +1847,9 @@ class Domain(TimeStampedModel, DomainHelper): missingSecurity = True missingTech = True - # Potential collision - mismatch between _contacts and contacts? - # But the ID wouldnt match in this case because the default is being grabbed? - if len(cleaned.get("_contacts")) < 3: - for contact in cleaned.get("_contacts"): + contacts = cleaned.get("_contacts", []) + if len(contacts) < 3: + for contact in contacts: if contact.type == PublicContact.ContactTypeChoices.ADMINISTRATIVE: missingAdmin = False if contact.type == PublicContact.ContactTypeChoices.SECURITY: @@ -1868,9 +1867,9 @@ class Domain(TimeStampedModel, DomainHelper): if missingTech: technical_contact = self.get_default_technical_contact() technical_contact.save() - + logger.info( - "_add_missing_contacts_if_unknown => In function. Values are " + "_add_missing_contacts_if_unknown => Adding contacts. Values are " f"missingAdmin: {missingAdmin}, missingSecurity: {missingSecurity}, missingTech: {missingTech}" ) @@ -2052,7 +2051,7 @@ class Domain(TimeStampedModel, DomainHelper): if contacts and isinstance(contacts, list) and len(contacts) > 0: cleaned_contacts = self._fetch_contacts(contacts) return cleaned_contacts - + def _fetch_contacts(self, contact_data): """Fetch contact info.""" choices = PublicContact.ContactTypeChoices @@ -2086,13 +2085,11 @@ class Domain(TimeStampedModel, DomainHelper): def _get_or_create_public_contact(self, public_contact: PublicContact): """Tries to find a PublicContact object in our DB. If it can't, it'll create it. Returns PublicContact""" - logger.info(f"in function") db_contact = PublicContact.objects.filter( registry_id=public_contact.registry_id, contact_type=public_contact.contact_type, domain=self, ) - logger.info(f"db_contact {db_contact}") # If we find duplicates, log it and delete the oldest ones. if db_contact.count() > 1: @@ -2119,13 +2116,10 @@ class Domain(TimeStampedModel, DomainHelper): with transaction.atomic(): public_contact.save(skip_epp_save=True) logger.info(f"Created a new PublicContact: {public_contact}") - # In rare cases, _add_missing_contacts_if_unknown will cause a race condition with this function. - # This is because it calls .save(), which is called here. - # except IntegrityError as err: logger.error( - "_get_or_create_public_contact() => tried to create a duplicate public contact: " - f"{err}", exc_info=True + "_get_or_create_public_contact() => tried to create a duplicate public contact: " f"{err}", + exc_info=True, ) return PublicContact.objects.get( registry_id=public_contact.registry_id, @@ -2142,7 +2136,7 @@ class Domain(TimeStampedModel, DomainHelper): if existing_contact.email != public_contact.email or existing_contact.registry_id != public_contact.registry_id: existing_contact.delete() public_contact.save() - logger.warning("Requested PublicContact is out of sync " "with DB.") + logger.warning("Requested PublicContact is out of sync with our DB.") return public_contact # If it already exists, we can assume that the DB instance was updated during set, so we should just use that. diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 8fde89fd0..083725a55 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -350,27 +350,28 @@ class TestDomainCreation(MockEppLib): """Rule: An approved domain request must result in a domain""" @less_console_noise_decorator - def test_get_security_email_race_condition(self): + def test_get_or_create_public_contact_race_condition(self): """ Scenario: Two processes try to create the same security contact simultaneously Given a domain in UNKNOWN state When a race condition occurs during contact creation Then no IntegrityError is raised And only one security contact exists in database - And the correct security email is returned + And the correct public contact is returned + + CONTEXT: We ran into an intermittent but somewhat rare issue where IntegrityError + was raised when creating PublicContact. + Per our logs, this seemed to appear during periods of high app activity. """ domain, _ = Domain.objects.get_or_create(name="defaultsecurity.gov") - - # Store original method - original_filter = PublicContact.objects.filter - self.first_call = True - def mock_filter(*args, **kwargs): - """ Simulates the race condition by creating a - duplicate contact between filter check and save - """ - result = original_filter(*args, **kwargs) - # Return empty queryset for first call. Otherwise just proceed as normal. + self.first_call = True + + def mock_filter(*args, **kwargs): + """Simulates a race condition by creating a + duplicate contact between the first filter and save. + """ + # Return an empty queryset for the first call. Otherwise just proceed as normal. if self.first_call: self.first_call = False duplicate = PublicContact( @@ -378,21 +379,21 @@ class TestDomainCreation(MockEppLib): contact_type=PublicContact.ContactTypeChoices.SECURITY, registry_id="defaultSec", email="dotgov@cisa.dhs.gov", - name="Registry Customer Service" + name="Registry Customer Service", ) duplicate.save(skip_epp_save=True) return PublicContact.objects.none() - return result - - with patch.object(PublicContact.objects, 'filter', side_effect=mock_filter): + return PublicContact.objects.filter(*args, **kwargs) + + with patch.object(PublicContact.objects, "filter", side_effect=mock_filter): try: public_contact = PublicContact( domain=domain, contact_type=PublicContact.ContactTypeChoices.SECURITY, registry_id="defaultSec", email="dotgov@cisa.dhs.gov", - name="Registry Customer Service" + name="Registry Customer Service", ) returned_public_contact = domain._get_or_create_public_contact(public_contact) except IntegrityError: @@ -403,13 +404,14 @@ class TestDomainCreation(MockEppLib): "Expected behavior: gracefully handle duplicate creation and return existing contact." ) - # Verify only one contact exists + # Verify that only one contact exists and its correctness security_contacts = PublicContact.objects.filter( - domain=domain, - contact_type=PublicContact.ContactTypeChoices.SECURITY + domain=domain, contact_type=PublicContact.ContactTypeChoices.SECURITY ) self.assertEqual(security_contacts.count(), 1) self.assertEqual(returned_public_contact, security_contacts.get()) + self.assertEqual(returned_public_contact.registry_id, "defaultSec") + self.assertEqual(returned_public_contact.email, "dotgov@cisa.dhs.gov") @boto3_mocking.patching def test_approved_domain_request_creates_domain_locally(self): From 8c204d4e8e2c392baddd35508dad83387a786358 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:10:24 -0700 Subject: [PATCH 189/201] Remove unrelated changes --- src/registrar/admin.py | 3 -- src/registrar/models/domain.py | 56 +++++++++++++++++----------------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 8773f7ef8..849cb6100 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -3835,9 +3835,6 @@ class PublicContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): change_form_template = "django/admin/email_clipboard_change_form.html" autocomplete_fields = ["domain"] - list_display = ("name", "contact_type", "email", "domain", "registry_id") - search_fields = ["email", "name", "registry_id"] - search_help_text = "Search by email, name or registry id." def changeform_view(self, request, object_id=None, form_url="", extra_context=None): if extra_context is None: diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 2ace5f22d..d024efe5f 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1665,6 +1665,30 @@ class Domain(TimeStampedModel, DomainHelper): ) return err.code + def _fetch_contacts(self, contact_data): + """Fetch contact info.""" + choices = PublicContact.ContactTypeChoices + # We expect that all these fields get populated, + # so we can create these early, rather than waiting. + contacts_dict = { + choices.ADMINISTRATIVE: None, + choices.SECURITY: None, + choices.TECHNICAL: None, + } + for domainContact in contact_data: + req = commands.InfoContact(id=domainContact.contact) + data = registry.send(req, cleaned=True).res_data[0] + logger.info(f"_fetch_contacts => this is the data: {data}") + + # Map the object we recieved from EPP to a PublicContact + mapped_object = self.map_epp_contact_to_public_contact(data, domainContact.contact, domainContact.type) + logger.info(f"_fetch_contacts => mapped_object: {mapped_object}") + + # Find/create it in the DB + in_db = self._get_or_create_public_contact(mapped_object) + contacts_dict[in_db.contact_type] = in_db.registry_id + return contacts_dict + def _get_or_create_contact(self, contact: PublicContact): """Try to fetch info about a contact. Create it if it does not exist.""" logger.info("_get_or_create_contact() -> Fetching contact info") @@ -1826,6 +1850,7 @@ class Domain(TimeStampedModel, DomainHelper): """ try: self._add_missing_contacts_if_unknown(cleaned) + except Exception as e: logger.error( "%s couldn't _add_missing_contacts_if_unknown, error was %s." @@ -1843,6 +1868,7 @@ class Domain(TimeStampedModel, DomainHelper): is in an UNKNOWN state, that is an error state) Note: The transition state change happens at the end of the function """ + missingAdmin = True missingSecurity = True missingTech = True @@ -1938,8 +1964,6 @@ class Domain(TimeStampedModel, DomainHelper): Additionally, capture and cache old hosts and contacts from cache if they don't exist in cleaned """ - - # object reference issue between self._cache vs cleaned? old_cache_hosts = self._cache.get("hosts") old_cache_contacts = self._cache.get("contacts") @@ -2052,30 +2076,6 @@ class Domain(TimeStampedModel, DomainHelper): cleaned_contacts = self._fetch_contacts(contacts) return cleaned_contacts - def _fetch_contacts(self, contact_data): - """Fetch contact info.""" - choices = PublicContact.ContactTypeChoices - # We expect that all these fields get populated, - # so we can create these early, rather than waiting. - contacts_dict = { - choices.ADMINISTRATIVE: None, - choices.SECURITY: None, - choices.TECHNICAL: None, - } - for domainContact in contact_data: - req = commands.InfoContact(id=domainContact.contact) - data = registry.send(req, cleaned=True).res_data[0] - logger.info(f"_fetch_contacts => this is the data: {data}") - - # Map the object we recieved from EPP to a PublicContact - mapped_object = self.map_epp_contact_to_public_contact(data, domainContact.contact, domainContact.type) - logger.info(f"_fetch_contacts => mapped_object: {mapped_object}") - - # Find/create it in the DB - in_db = self._get_or_create_public_contact(mapped_object) - contacts_dict[in_db.contact_type] = in_db.registry_id - return contacts_dict - def _get_hosts(self, hosts): cleaned_hosts = [] if hosts and isinstance(hosts, list): @@ -2118,7 +2118,7 @@ class Domain(TimeStampedModel, DomainHelper): logger.info(f"Created a new PublicContact: {public_contact}") except IntegrityError as err: logger.error( - "_get_or_create_public_contact() => tried to create a duplicate public contact: " f"{err}", + f"_get_or_create_public_contact() => tried to create a duplicate public contact: {err}", exc_info=True, ) return PublicContact.objects.get( @@ -2136,7 +2136,7 @@ class Domain(TimeStampedModel, DomainHelper): if existing_contact.email != public_contact.email or existing_contact.registry_id != public_contact.registry_id: existing_contact.delete() public_contact.save() - logger.warning("Requested PublicContact is out of sync with our DB.") + logger.warning("Requested PublicContact is out of sync with DB.") return public_contact # If it already exists, we can assume that the DB instance was updated during set, so we should just use that. From 016c0851419abeacefaa7c726581beab0a6ca668 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Fri, 17 Jan 2025 15:33:41 -0600 Subject: [PATCH 190/201] Change log_change --- src/registrar/views/transfer_user.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index fa66185ca..ea9da33c3 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -118,7 +118,8 @@ class TransferUserView(View): with ignore_unique_violation(): setattr(related_object, related_field.field.name, current_user) related_object.save() - self.log_change(selected_user, current_user, related_field.field.name, change_logs) + obj_type = related_field.related_model.__name__ + self.log_change(obj_type, selected_user, current_user, related_field.field.name, change_logs) def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): # Handle ForeignKey relationships @@ -126,7 +127,7 @@ class TransferUserView(View): if related_object: setattr(current_user, related_field.name, related_object) current_user.save() - self.log_change(selected_user, current_user, related_field.name, change_logs) + self.log_change(related_object, selected_user, current_user, related_field.name, change_logs) def _handle_one_to_one(self, related_field: OneToOneField, selected_user, current_user, change_logs): # Handle OneToOne relationship @@ -135,7 +136,7 @@ class TransferUserView(View): with ignore_unique_violation(): setattr(current_user, related_field.name, related_object) current_user.save() - self.log_change(selected_user, current_user, related_field.name, change_logs) + self.log_change(related_object.__name__, selected_user, current_user, related_field.name, change_logs) def _handle_many_to_many(self, related_field: ManyToManyField, selected_user, current_user, change_logs): # Handle ManyToMany relationship @@ -146,7 +147,8 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - self.log_change(selected_user, current_user, related_name, change_logs) + obj_type = related_field.related_model.__name__ + self.log_change(obj_type, selected_user, current_user, related_name, change_logs) def _handle_many_to_many_reverse(self, related_field: ManyToManyRel, selected_user, current_user, change_logs): # Handle reverse relationship @@ -157,7 +159,8 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - self.log_change(selected_user, current_user, related_name, change_logs) + obj_type = related_field.related_model.__name__ + self.log_change(obj_type, selected_user, current_user, related_name, change_logs) def _handle_one_to_one_reverse(self, related_field: OneToOneRel, selected_user, current_user, change_logs): # Handle reverse relationship @@ -166,11 +169,11 @@ class TransferUserView(View): if related_instance: setattr(related_instance, field_name, current_user) related_instance.save() - self.log_change(selected_user, current_user, field_name, change_logs) + self.log_change(related_instance.__name__, selected_user, current_user, field_name, change_logs) @classmethod - def log_change(cls, selected_user, current_user, field_name, change_logs): - log_entry = f"Transferred {field_name} from {selected_user} to {current_user}" + def log_change(cls, obj, selected_user, current_user, field_name, change_logs): + log_entry = f"Changed {field_name} from {selected_user} to {current_user} on {obj}" logger.info(log_entry) change_logs.append(log_entry) From ba2788014b5118364c1c0054b3587e68e56d7601 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Jan 2025 17:38:00 -0500 Subject: [PATCH 191/201] removed default empty options from Checkboxes as they are only needed for Selects --- src/registrar/assets/src/js/getgov/requesting-entity.js | 3 +-- src/registrar/forms/domain.py | 3 +-- src/registrar/forms/domain_request_wizard.py | 9 +++------ src/registrar/forms/portfolio.py | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/registrar/assets/src/js/getgov/requesting-entity.js b/src/registrar/assets/src/js/getgov/requesting-entity.js index 5f3be8c79..833eab2f8 100644 --- a/src/registrar/assets/src/js/getgov/requesting-entity.js +++ b/src/registrar/assets/src/js/getgov/requesting-entity.js @@ -15,7 +15,6 @@ export function handleRequestingEntityFieldset() { const suborgContainer = document.getElementById("suborganization-container"); const suborgDetailsContainer = document.getElementById("suborganization-container__details"); const suborgAddtlInstruction = document.getElementById("suborganization-addtl-instruction"); - const subOrgCreateNewOption = document.getElementById("option-to-add-suborg")?.value; // Make sure all crucial page elements exist before proceeding. // This more or less ensures that we are on the Requesting Entity page, and not elsewhere. if (!radios || !input || !select || !inputGrandParent || !suborgContainer || !suborgDetailsContainer) return; @@ -28,7 +27,7 @@ export function handleRequestingEntityFieldset() { function toggleSuborganization(radio=null) { if (radio != null) requestingSuborganization = radio?.checked && radio.value === "True"; requestingSuborganization ? showElement(suborgContainer) : hideElement(suborgContainer); - if (select.options.length == 2) { // --Select-- and other are the only options + if (select.options.length == 1) { // other is the only option hideElement(inputGrandParent); // Hide the combo box and indicate requesting new suborg hideElement(suborgAddtlInstruction); // Hide additional instruction related to the list requestingNewSuborganization.value = "True"; diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index b3dae0d3a..5eeae232d 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -450,7 +450,6 @@ class DomainOrgNameAddressForm(forms.ModelForm): label="Federal agency", required=False, queryset=FederalAgency.objects.all(), - empty_label="--Select--", widget=ComboboxWidget, ) zipcode = forms.CharField( @@ -469,7 +468,7 @@ class DomainOrgNameAddressForm(forms.ModelForm): state_territory = forms.ChoiceField( label="State, territory, or military post", required=True, - choices=[("", "--Select--")] + DomainInformation.StateTerritoryChoices.choices, + choices=DomainInformation.StateTerritoryChoices.choices, error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 12d3b74bf..7f80c9d34 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -44,7 +44,6 @@ class RequestingEntityForm(RegistrarForm): label="Suborganization name", required=False, queryset=Suborganization.objects.none(), - empty_label="--Select--", widget=ComboboxWidget, ) requested_suborganization = forms.CharField( @@ -58,7 +57,7 @@ class RequestingEntityForm(RegistrarForm): suborganization_state_territory = forms.ChoiceField( label="State, territory, or military post", required=False, - choices=[("", "--Select--")] + DomainRequest.StateTerritoryChoices.choices, + choices=DomainRequest.StateTerritoryChoices.choices, widget=ComboboxWidget, ) @@ -74,8 +73,7 @@ class RequestingEntityForm(RegistrarForm): # Modify the choices to include "other" so that form can display options properly self.fields["sub_organization"].choices = ( - [("", "--Select--")] - + [(obj.id, str(obj)) for obj in queryset] + [(obj.id, str(obj)) for obj in queryset] + [("other", "Other (enter your suborganization manually)")] ) @@ -328,7 +326,6 @@ class OrganizationContactForm(RegistrarForm): # uncomment to see if modelChoiceField can be an arg later required=False, queryset=FederalAgency.objects.exclude(agency__in=excluded_agencies), - empty_label="--Select--", widget=ComboboxWidget, ) organization_name = forms.CharField( @@ -349,7 +346,7 @@ class OrganizationContactForm(RegistrarForm): ) state_territory = forms.ChoiceField( label="State, territory, or military post", - choices=[("", "--Select--")] + DomainRequest.StateTerritoryChoices.choices, + choices=DomainRequest.StateTerritoryChoices.choices, error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, diff --git a/src/registrar/forms/portfolio.py b/src/registrar/forms/portfolio.py index 13d956fe4..9d6c9ecdf 100644 --- a/src/registrar/forms/portfolio.py +++ b/src/registrar/forms/portfolio.py @@ -37,7 +37,7 @@ class PortfolioOrgAddressForm(forms.ModelForm): state_territory = forms.ChoiceField( label="State, territory, or military post", required=True, - choices=[("", "--Select--")] + DomainInformation.StateTerritoryChoices.choices, + choices=DomainInformation.StateTerritoryChoices.choices, error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, From 463632baf769c901c1cec198fdcda293744d632d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Jan 2025 17:42:00 -0500 Subject: [PATCH 192/201] lint --- src/registrar/forms/domain_request_wizard.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index 7f80c9d34..e67595c21 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -72,10 +72,9 @@ class RequestingEntityForm(RegistrarForm): self.fields["sub_organization"].queryset = queryset # Modify the choices to include "other" so that form can display options properly - self.fields["sub_organization"].choices = ( - [(obj.id, str(obj)) for obj in queryset] - + [("other", "Other (enter your suborganization manually)")] - ) + self.fields["sub_organization"].choices = [(obj.id, str(obj)) for obj in queryset] + [ + ("other", "Other (enter your suborganization manually)") + ] @classmethod def from_database(cls, obj: DomainRequest | Contact | None): From eca4cd003aaa47855d580b8577cf41a21ddff2b2 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Fri, 17 Jan 2025 17:03:13 -0600 Subject: [PATCH 193/201] move logs into loops for some helper functions --- src/registrar/views/transfer_user.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index ea9da33c3..d030717b4 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -118,8 +118,7 @@ class TransferUserView(View): with ignore_unique_violation(): setattr(related_object, related_field.field.name, current_user) related_object.save() - obj_type = related_field.related_model.__name__ - self.log_change(obj_type, selected_user, current_user, related_field.field.name, change_logs) + self.log_change(related_object.__name__, selected_user, current_user, related_field.field.name, change_logs) def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): # Handle ForeignKey relationships @@ -147,8 +146,7 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - obj_type = related_field.related_model.__name__ - self.log_change(obj_type, selected_user, current_user, related_name, change_logs) + self.log_change(instance.__name__, selected_user, current_user, related_name, change_logs) def _handle_many_to_many_reverse(self, related_field: ManyToManyRel, selected_user, current_user, change_logs): # Handle reverse relationship @@ -159,8 +157,7 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - obj_type = related_field.related_model.__name__ - self.log_change(obj_type, selected_user, current_user, related_name, change_logs) + self.log_change(instance.__name__, selected_user, current_user, related_name, change_logs) def _handle_one_to_one_reverse(self, related_field: OneToOneRel, selected_user, current_user, change_logs): # Handle reverse relationship From 9d8755f28052ce31045980f8323260f5db26982a Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Fri, 17 Jan 2025 17:16:24 -0600 Subject: [PATCH 194/201] log objects instead of names --- src/registrar/views/transfer_user.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/views/transfer_user.py b/src/registrar/views/transfer_user.py index d030717b4..f574b76d9 100644 --- a/src/registrar/views/transfer_user.py +++ b/src/registrar/views/transfer_user.py @@ -118,7 +118,7 @@ class TransferUserView(View): with ignore_unique_violation(): setattr(related_object, related_field.field.name, current_user) related_object.save() - self.log_change(related_object.__name__, selected_user, current_user, related_field.field.name, change_logs) + self.log_change(related_object, selected_user, current_user, related_field.field.name, change_logs) def _handle_foreign_key(self, related_field: ForeignKey, selected_user, current_user, change_logs): # Handle ForeignKey relationships @@ -135,7 +135,7 @@ class TransferUserView(View): with ignore_unique_violation(): setattr(current_user, related_field.name, related_object) current_user.save() - self.log_change(related_object.__name__, selected_user, current_user, related_field.name, change_logs) + self.log_change(related_object, selected_user, current_user, related_field.name, change_logs) def _handle_many_to_many(self, related_field: ManyToManyField, selected_user, current_user, change_logs): # Handle ManyToMany relationship @@ -146,7 +146,7 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - self.log_change(instance.__name__, selected_user, current_user, related_name, change_logs) + self.log_change(instance, selected_user, current_user, related_name, change_logs) def _handle_many_to_many_reverse(self, related_field: ManyToManyRel, selected_user, current_user, change_logs): # Handle reverse relationship @@ -157,7 +157,7 @@ class TransferUserView(View): with ignore_unique_violation(): getattr(instance, related_name).remove(selected_user) getattr(instance, related_name).add(current_user) - self.log_change(instance.__name__, selected_user, current_user, related_name, change_logs) + self.log_change(instance, selected_user, current_user, related_name, change_logs) def _handle_one_to_one_reverse(self, related_field: OneToOneRel, selected_user, current_user, change_logs): # Handle reverse relationship @@ -166,7 +166,7 @@ class TransferUserView(View): if related_instance: setattr(related_instance, field_name, current_user) related_instance.save() - self.log_change(related_instance.__name__, selected_user, current_user, field_name, change_logs) + self.log_change(related_instance, selected_user, current_user, field_name, change_logs) @classmethod def log_change(cls, obj, selected_user, current_user, field_name, change_logs): From 96618ee2bc6ff7d1a2a9cee33cac68d38ec10679 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Jan 2025 20:03:04 -0500 Subject: [PATCH 195/201] fixing tests --- src/registrar/forms/domain_request_wizard.py | 3 -- src/registrar/tests/test_views_portfolio.py | 30 +++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/registrar/forms/domain_request_wizard.py b/src/registrar/forms/domain_request_wizard.py index e67595c21..7c9dcb180 100644 --- a/src/registrar/forms/domain_request_wizard.py +++ b/src/registrar/forms/domain_request_wizard.py @@ -95,7 +95,6 @@ class RequestingEntityForm(RegistrarForm): def clean_sub_organization(self): """On suborganization clean, set the suborganization value to None if the user is requesting a custom suborganization (as it doesn't exist yet)""" - # If it's a new suborganization, return None (equivalent to selecting nothing) if self.cleaned_data.get("is_requesting_new_suborganization"): return None @@ -129,7 +128,6 @@ class RequestingEntityForm(RegistrarForm): # Retrieve sub_organization and store in _original_suborganization suborganization = data.get("portfolio_requesting_entity-sub_organization") self._original_suborganization = suborganization - # If the original value was "other", clear it for validation if self._original_suborganization == "other": data["portfolio_requesting_entity-sub_organization"] = "" @@ -171,7 +169,6 @@ class RequestingEntityForm(RegistrarForm): requesting_entity_is_suborganization = self.data.get( "portfolio_requesting_entity-requesting_entity_is_suborganization" ) - if requesting_entity_is_suborganization == "True": if is_requesting_new_suborganization: if not cleaned_data.get("requested_suborganization") and "requested_suborganization" not in self.errors: diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 78a4dae82..e2de3328f 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2879,7 +2879,7 @@ class TestRequestingEntity(WebTest): form["portfolio_requesting_entity-requesting_entity_is_suborganization"] = True form["portfolio_requesting_entity-is_requesting_new_suborganization"] = True - form["portfolio_requesting_entity-sub_organization"] = "" + form["portfolio_requesting_entity-sub_organization"] = "other" form["portfolio_requesting_entity-requested_suborganization"] = "moon" form["portfolio_requesting_entity-suborganization_city"] = "kepler" @@ -2933,7 +2933,7 @@ class TestRequestingEntity(WebTest): @override_flag("organization_feature", active=True) @override_flag("organization_requests", active=True) - @less_console_noise_decorator + # @less_console_noise_decorator def test_requesting_entity_page_errors(self): """Tests that we get the expected form errors on requesting entity""" domain_request = completed_domain_request(user=self.user, portfolio=self.portfolio) @@ -2942,18 +2942,34 @@ class TestRequestingEntity(WebTest): session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # For 2 the tests below, it is required to submit a form without submitting a value + # for the select/combobox. WebTest will not do this; by default, WebTest will submit + # the first choice in a select. So, need to manipulate the form to remove the + # particular select/combobox that will not be submitted, and then post the form. + form_action = f"/request/{domain_request.pk}/portfolio_requesting_entity/" + # Test missing suborganization selection form["portfolio_requesting_entity-requesting_entity_is_suborganization"] = True - form["portfolio_requesting_entity-sub_organization"] = "" - - response = form.submit() + form["portfolio_requesting_entity-is_requesting_new_suborganization"] = False + # remove sub_organization from the form submission + form_data = form.submit_fields() + form_data = [(key, value) for key, value in form_data if key != "portfolio_requesting_entity-sub_organization"] + response = self.app.post(form_action, dict(form_data)) self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) self.assertContains(response, "Suborganization is required.", status_code=200) # Test missing custom suborganization details + form["portfolio_requesting_entity-requesting_entity_is_suborganization"] = True form["portfolio_requesting_entity-is_requesting_new_suborganization"] = True - response = form.submit() - self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + form["portfolio_requesting_entity-sub_organization"] = "other" + # remove suborganization_state_territory from the form submission + form_data = form.submit_fields() + form_data = [ + (key, value) + for key, value in form_data + if key != "portfolio_requesting_entity-suborganization_state_territory" + ] + response = self.app.post(form_action, dict(form_data)) self.assertContains(response, "Enter the name of your suborganization.", status_code=200) self.assertContains(response, "Enter the city where your suborganization is located.", status_code=200) self.assertContains( From 6bf52f00150daeaee062c795382cb8f07282c0f7 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 20 Jan 2025 07:10:12 -0500 Subject: [PATCH 196/201] cleanup --- src/registrar/forms/domain.py | 1 - src/registrar/tests/test_views_portfolio.py | 2 +- src/registrar/views/domain_request.py | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 5eeae232d..711315632 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -164,7 +164,6 @@ class DomainSuborganizationForm(forms.ModelForm): sub_organization = forms.ModelChoiceField( label="Suborganization name", queryset=Suborganization.objects.none(), - empty_label="⎯ (No suborganization)", required=False, widget=ComboboxWidget, ) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index e2de3328f..69502d683 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2933,7 +2933,7 @@ class TestRequestingEntity(WebTest): @override_flag("organization_feature", active=True) @override_flag("organization_requests", active=True) - # @less_console_noise_decorator + @less_console_noise_decorator def test_requesting_entity_page_errors(self): """Tests that we get the expected form errors on requesting entity""" domain_request = completed_domain_request(user=self.user, portfolio=self.portfolio) diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index bff3e5c00..9754b0d0c 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -368,6 +368,7 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView): and from the database if `use_db` is True (provided that record exists). An empty form will be provided if neither of those are true. """ + kwargs = { "files": files, "prefix": self.steps.current, From 6905531061e48eb3f24b70f0d8c64982e9b3b34a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 20 Jan 2025 07:34:00 -0500 Subject: [PATCH 197/201] fixing err message, and updating comment --- src/registrar/utility/email_invitations.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 3653d4290..25e9db0f3 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -52,6 +52,11 @@ def send_domain_invitation_email( def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, requested_user=None): + """ + Notifies all domain managers of the provided domain of a change + Raises: + EmailSendingError + """ # Get each domain manager from list user_domain_roles = UserDomainRole.objects.filter(domain=domain) for user_domain_role in user_domain_roles: @@ -72,7 +77,7 @@ def send_emails_to_domain_managers(email: str, requestor_email, domain: Domain, ) except EmailSendingError as err: raise EmailSendingError( - f"Could not send email manager notification to {user.email} for domains: {domain.name}" + f"Could not send email manager notification to {user.email} for domain: {domain.name}" ) from err From 9b6f637270edcf96f74aa00e125cc7e4b0473c2e Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Tue, 21 Jan 2025 10:45:19 -0600 Subject: [PATCH 198/201] fix unit test --- src/registrar/tests/test_admin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 8baf5e42d..d58ee59a2 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -2866,7 +2866,7 @@ class TestTransferUser(WebTest): with self.assertRaises(User.DoesNotExist): self.user2.refresh_from_db() - @less_console_noise_decorator + # @less_console_noise_decorator def test_transfer_user_throws_transfer_and_delete_success_messages(self): """Test that success messages for data transfer and user deletion are displayed.""" # Ensure the setup for VerifiedByStaff @@ -2884,11 +2884,13 @@ class TestTransferUser(WebTest): self.assertContains(after_submit, "

        Change user

        ") + print(mock_success_message.call_args_list) + mock_success_message.assert_any_call( ANY, ( - "Data transferred successfully for the following objects: ['Transferred requestor " - + "from Furiosa Jabassa to Max Rokatanski ']" + "Data transferred successfully for the following objects: ['Changed requestor " + + "from Furiosa Jabassa to Max Rokatanski on immortan.joe@citadel.com']" ), ) From e9268bea8b3883a6ae62d578617b168c5f36d107 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Jan 2025 13:17:22 -0500 Subject: [PATCH 199/201] updated required attribute on state territory --- src/registrar/forms/domain.py | 2 +- src/registrar/forms/portfolio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 711315632..20091a325 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -471,7 +471,7 @@ class DomainOrgNameAddressForm(forms.ModelForm): error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, - widget=ComboboxWidget(), + widget=ComboboxWidget(attrs={"required":True}), ) class Meta: diff --git a/src/registrar/forms/portfolio.py b/src/registrar/forms/portfolio.py index 9d6c9ecdf..379e8053c 100644 --- a/src/registrar/forms/portfolio.py +++ b/src/registrar/forms/portfolio.py @@ -41,7 +41,7 @@ class PortfolioOrgAddressForm(forms.ModelForm): error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, - widget=ComboboxWidget, + widget=ComboboxWidget(attrs={"required":True}), ) class Meta: From 5a30d28b34307e62432e8a7ed841baaa921b03a6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:20:11 -0700 Subject: [PATCH 200/201] cleanup logs --- src/registrar/models/domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index d024efe5f..cb481db7a 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1329,14 +1329,14 @@ class Domain(TimeStampedModel, DomainHelper): def get_default_administrative_contact(self): """Gets the default administrative contact.""" - logger.info("get_default_administrative_contact() -> Adding administrative security contact") + logger.info("get_default_administrative_contact() -> Adding default administrative contact") contact = PublicContact.get_default_administrative() contact.domain = self return contact def get_default_technical_contact(self): """Gets the default technical contact.""" - logger.info("get_default_security_contact() -> Adding technical security contact") + logger.info("get_default_security_contact() -> Adding default technical contact") contact = PublicContact.get_default_technical() contact.domain = self return contact From 6828a1f1ff022a4da0c58b8e17f34d1d6320f439 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Jan 2025 15:33:08 -0500 Subject: [PATCH 201/201] lint --- src/registrar/forms/domain.py | 2 +- src/registrar/forms/portfolio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 20091a325..05eb90db3 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -471,7 +471,7 @@ class DomainOrgNameAddressForm(forms.ModelForm): error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, - widget=ComboboxWidget(attrs={"required":True}), + widget=ComboboxWidget(attrs={"required": True}), ) class Meta: diff --git a/src/registrar/forms/portfolio.py b/src/registrar/forms/portfolio.py index 379e8053c..e57b56c4f 100644 --- a/src/registrar/forms/portfolio.py +++ b/src/registrar/forms/portfolio.py @@ -41,7 +41,7 @@ class PortfolioOrgAddressForm(forms.ModelForm): error_messages={ "required": ("Select the state, territory, or military post where your organization is located.") }, - widget=ComboboxWidget(attrs={"required":True}), + widget=ComboboxWidget(attrs={"required": True}), ) class Meta: