From c2d17442f8a05ab7c70929c954519352ab731dcc Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 Jan 2025 17:04:02 -0500 Subject: [PATCH 01/58] domain invitations now send email from django admin for both domain and portfolio invitations, consolidated error handling --- src/registrar/admin.py | 145 +++++++++++++---- src/registrar/models/domain_invitation.py | 2 + src/registrar/utility/email_invitations.py | 52 +++--- src/registrar/utility/errors.py | 13 +- src/registrar/views/domain.py | 148 ++++-------------- .../views/utility/portfolio_helper.py | 83 ++++++++++ 6 files changed, 264 insertions(+), 179 deletions(-) create mode 100644 src/registrar/views/utility/portfolio_helper.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 849cb6100..183d251b4 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -14,6 +14,7 @@ from django.db.models import ( from django.db.models.functions import Concat, Coalesce from django.http import HttpResponseRedirect from registrar.models.federal_agency import FederalAgency +from registrar.models.portfolio_invitation import PortfolioInvitation from registrar.utility.admin_helpers import ( AutocompleteSelectWithPlaceholder, get_action_needed_reason_default_email, @@ -27,8 +28,12 @@ from django.shortcuts import redirect from django_fsm import get_available_FIELD_transitions, FSMField from registrar.models import DomainInformation, Portfolio, UserPortfolioPermission, DomainInvitation from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices -from registrar.utility.email import EmailSendingError -from registrar.utility.email_invitations import send_portfolio_invitation_email +from registrar.utility.email_invitations import send_domain_invitation_email, send_portfolio_invitation_email +from registrar.views.utility.portfolio_helper import ( + get_org_membership, + get_requested_user, + handle_invitation_exceptions, +) from waffle.decorators import flag_is_active from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin @@ -41,7 +46,7 @@ from waffle.admin import FlagAdmin from waffle.models import Sample, Switch from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website, SeniorOfficial from registrar.utility.constants import BranchChoices -from registrar.utility.errors import FSMDomainRequestError, FSMErrorCodes, MissingEmailError +from registrar.utility.errors import FSMDomainRequestError, FSMErrorCodes from registrar.utility.waffle import flag_is_active_for_user from registrar.views.utility.mixins import OrderableFieldsMixin from django.contrib.admin.views.main import ORDER_VAR @@ -1442,11 +1447,108 @@ class DomainInvitationAdmin(ListHeaderAdmin): which will be successful if a single User exists for that email; otherwise, will just continue to create the invitation. """ - if not change and User.objects.filter(email=obj.email).count() == 1: - # Domain Invitation creation for an existing User - obj.retrieve() - # Call the parent save method to save the object - super().save_model(request, obj, form, change) + if not change: + domain = obj.domain + domain_org = domain.domain_info.portfolio + requested_email = obj.email + # Look up a user with that email + requested_user = get_requested_user(requested_email) + requestor = request.user + + member_of_a_different_org, member_of_this_org = get_org_membership( + domain_org, requested_email, requested_user + ) + + try: + if ( + flag_is_active(request, "organization_feature") + and not flag_is_active(request, "multiple_portfolios") + and domain_org is not None + and not member_of_this_org + ): + send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) + PortfolioInvitation.objects.get_or_create( + email=requested_email, + portfolio=domain_org, + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], + ) + messages.success(request, f"{requested_email} has been invited to the organization: {domain_org}") + + send_domain_invitation_email( + email=requested_email, + requestor=requestor, + domain=domain, + is_member_of_different_org=member_of_a_different_org, + ) + if requested_user is not None: + # Domain Invitation creation for an existing User + obj.retrieve() + # Call the parent save method to save the object + super().save_model(request, obj, form, change) + messages.success(request, f"{requested_email} has been invited to the domain: {domain}") + except Exception as e: + handle_invitation_exceptions(request, e, requested_email) + return + else: + # Call the parent save method to save the object + super().save_model(request, obj, form, change) + + def response_add(self, request, obj, post_url_continue=None): + """ + Override response_add to handle rendering when exceptions are raised during add model. + + Normal flow on successful save_model on add is to redirect to changelist_view. + If there are errors, flow is modified to instead render change form. + """ + # Check if there are any error or warning messages in the `messages` framework + storage = get_messages(request) + has_errors = any(message.level_tag in ["error", "warning"] for message in storage) + + if has_errors: + # Re-render the change form if there are errors or warnings + # Prepare context for rendering the change form + + # Get the model form + ModelForm = self.get_form(request, obj=obj) + form = ModelForm(instance=obj) + + # Create an AdminForm instance + admin_form = AdminForm( + form, + list(self.get_fieldsets(request, obj)), + self.get_prepopulated_fields(request, obj), + self.get_readonly_fields(request, obj), + model_admin=self, + ) + media = self.media + form.media + + opts = obj._meta + change_form_context = { + **self.admin_site.each_context(request), # Add admin context + "title": f"Add {opts.verbose_name}", + "opts": opts, + "original": obj, + "save_as": self.save_as, + "has_change_permission": self.has_change_permission(request, obj), + "add": True, # Indicate this is an "Add" form + "change": False, # Indicate this is not a "Change" form + "is_popup": False, + "inline_admin_formsets": [], + "save_on_top": self.save_on_top, + "show_delete": self.has_delete_permission(request, obj), + "obj": obj, + "adminform": admin_form, # Pass the AdminForm instance + "media": media, + "errors": None, + } + return self.render_change_form( + request, + context=change_form_context, + add=True, + change=False, + obj=obj, + ) + return super().response_add(request, obj, post_url_continue) class PortfolioInvitationAdmin(ListHeaderAdmin): @@ -1523,36 +1625,11 @@ class PortfolioInvitationAdmin(ListHeaderAdmin): messages.warning(request, "User is already a member of this portfolio.") except Exception as e: # when exception is raised, handle and do not save the model - self._handle_exceptions(e, request, obj) + handle_invitation_exceptions(request, e, requested_email) return # Call the parent save method to save the object super().save_model(request, obj, form, change) - def _handle_exceptions(self, exception, request, obj): - """Handle exceptions raised during the process. - - Log warnings / errors, and message errors to the user. - """ - if isinstance(exception, EmailSendingError): - logger.warning( - "Could not sent email invitation to %s for portfolio %s (EmailSendingError)", - obj.email, - obj.portfolio, - exc_info=True, - ) - messages.error(request, "Could not send email invitation. Portfolio invitation not saved.") - elif isinstance(exception, MissingEmailError): - messages.error(request, str(exception)) - logger.error( - f"Can't send email to '{obj.email}' for portfolio '{obj.portfolio}'. " - f"No email exists for the requestor.", - exc_info=True, - ) - - else: - logger.warning("Could not send email invitation (Other Exception)", exc_info=True) - messages.error(request, "Could not send email invitation. Portfolio invitation not saved.") - def response_add(self, request, obj, post_url_continue=None): """ Override response_add to handle rendering when exceptions are raised during add model. diff --git a/src/registrar/models/domain_invitation.py b/src/registrar/models/domain_invitation.py index 28089dcb5..e2aede696 100644 --- a/src/registrar/models/domain_invitation.py +++ b/src/registrar/models/domain_invitation.py @@ -56,6 +56,8 @@ class DomainInvitation(TimeStampedModel): Raises: RuntimeError if no matching user can be found. """ + # NOTE: this is currently not accounting for scenario when User.objects.get matches + # multiple user accounts with the same email address # get a user with this email address User = get_user_model() diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 7171b8902..9455c5927 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -7,7 +7,7 @@ from registrar.utility.errors import ( OutsideOrgMemberError, ) from registrar.utility.waffle import flag_is_active_for_user -from registrar.utility.email import send_templated_email +from registrar.utility.email import EmailSendingError, send_templated_email import logging logger = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif # Check if the requestor is staff and has an email if not requestor.is_staff: if not requestor.email or requestor.email.strip() == "": - raise MissingEmailError + raise MissingEmailError(email=email, domain=domain) else: requestor_email = requestor.email @@ -65,15 +65,18 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif pass # Send the email - send_templated_email( - "emails/domain_invitation.txt", - "emails/domain_invitation_subject.txt", - to_address=email, - context={ - "domain": domain, - "requestor_email": requestor_email, - }, - ) + try: + send_templated_email( + "emails/domain_invitation.txt", + "emails/domain_invitation_subject.txt", + to_address=email, + context={ + "domain": domain, + "requestor_email": requestor_email, + }, + ) + except EmailSendingError as err: + raise EmailSendingError(f"Could not send email invitation to {email} for domain {domain}.") from err def send_portfolio_invitation_email(email: str, requestor, portfolio): @@ -98,17 +101,22 @@ def send_portfolio_invitation_email(email: str, requestor, portfolio): # Check if the requestor is staff and has an email if not requestor.is_staff: if not requestor.email or requestor.email.strip() == "": - raise MissingEmailError + raise MissingEmailError(email=email, portfolio=portfolio) else: requestor_email = requestor.email - send_templated_email( - "emails/portfolio_invitation.txt", - "emails/portfolio_invitation_subject.txt", - to_address=email, - context={ - "portfolio": portfolio, - "requestor_email": requestor_email, - "email": email, - }, - ) + try: + send_templated_email( + "emails/portfolio_invitation.txt", + "emails/portfolio_invitation_subject.txt", + to_address=email, + context={ + "portfolio": portfolio, + "requestor_email": requestor_email, + "email": email, + }, + ) + except EmailSendingError as err: + raise EmailSendingError( + f"Could not sent email invitation to {email} for portfolio {portfolio}. Portfolio invitation not saved." + ) from err diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 039fb3696..cc18d7269 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -46,8 +46,17 @@ class AlreadyDomainInvitedError(InvitationError): class MissingEmailError(InvitationError): """Raised when the requestor has no email associated with their account.""" - def __init__(self): - super().__init__("Can't send invitation email. No email is associated with your user account.") + def __init__(self, email=None, domain=None, portfolio=None): + # Default message if no additional info is provided + message = "Can't send invitation email. No email is associated with your user account." + + # Customize message based on provided arguments + if email and domain: + message = f"Can't send email to '{email}' on domain '{domain}'. No email exists for the requestor." + elif email and portfolio: + message = f"Can't send email to '{email}' for portfolio '{portfolio}'. No email exists for the requestor." + + super().__init__(message) class OutsideOrgMemberError(ValueError): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index f544a20f7..14e7dc933 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -10,7 +10,6 @@ import logging import requests from django.contrib import messages from django.contrib.messages.views import SuccessMessageMixin -from django.db import IntegrityError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse @@ -31,22 +30,23 @@ from registrar.models.user_portfolio_permission import UserPortfolioPermission from registrar.models.utility.portfolio_helper import UserPortfolioRoleChoices from registrar.utility.enums import DefaultEmail from registrar.utility.errors import ( - AlreadyDomainInvitedError, - AlreadyDomainManagerError, GenericError, GenericErrorCodes, - MissingEmailError, NameserverError, NameserverErrorCodes as nsErrorCodes, DsDataError, DsDataErrorCodes, SecurityEmailError, SecurityEmailErrorCodes, - OutsideOrgMemberError, ) from registrar.models.utility.contact_error import ContactError from registrar.views.utility.permission_views import UserDomainRolePermissionDeleteView from registrar.utility.waffle import flag_is_active_for_user +from registrar.views.utility.portfolio_helper import ( + get_org_membership, + get_requested_user, + handle_invitation_exceptions, +) from ..forms import ( SeniorOfficialContactForm, @@ -1149,43 +1149,13 @@ class DomainAddUserView(DomainFormBaseView): def get_success_url(self): return reverse("domain-users", kwargs={"pk": self.object.pk}) - def _get_org_membership(self, requestor_org, requested_email, requested_user): - """ - Verifies if an email belongs to a different organization as a member or invited member. - Verifies if an email belongs to this organization as a member or invited member. - User does not belong to any org can be deduced from the tuple returned. - - Returns a tuple (member_of_a_different_org, member_of_this_org). - """ - - # COMMENT: this code does not take into account multiple portfolios flag - - # COMMENT: shouldn't this code be based on the organization of the domain, not the org - # of the requestor? requestor could have multiple portfolios - - # Check for existing permissions or invitations for the requested user - existing_org_permission = UserPortfolioPermission.objects.filter(user=requested_user).first() - existing_org_invitation = PortfolioInvitation.objects.filter(email=requested_email).first() - - # Determine membership in a different organization - member_of_a_different_org = ( - existing_org_permission and existing_org_permission.portfolio != requestor_org - ) or (existing_org_invitation and existing_org_invitation.portfolio != requestor_org) - - # Determine membership in the same organization - member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == requestor_org) or ( - existing_org_invitation and existing_org_invitation.portfolio == requestor_org - ) - - return member_of_a_different_org, member_of_this_org - def form_valid(self, form): """Add the specified user to this domain.""" requested_email = form.cleaned_data["email"] requestor = self.request.user # Look up a user with that email - requested_user = self._get_requested_user(requested_email) + requested_user = get_requested_user(requested_email) # NOTE: This does not account for multiple portfolios flag being set to True domain_org = self.object.domain_info.portfolio @@ -1196,49 +1166,36 @@ class DomainAddUserView(DomainFormBaseView): or requestor.is_staff ) - member_of_a_different_org, member_of_this_org = self._get_org_membership( - domain_org, requested_email, requested_user - ) - - # determine portfolio of the domain (code currently is looking at requestor's portfolio) - # if requested_email/user is not member or invited member of this portfolio - # COMMENT: this code does not take into account multiple portfolios flag - # send portfolio invitation email - # create portfolio invitation - # create message to view - if ( - flag_is_active_for_user(requestor, "organization_feature") - and not flag_is_active_for_user(requestor, "multiple_portfolios") - and domain_org is not None - and requestor_can_update_portfolio - and not member_of_this_org - ): - try: - send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) - PortfolioInvitation.objects.get_or_create(email=requested_email, portfolio=domain_org) - messages.success(self.request, f"{requested_email} has been invited to the organization: {domain_org}") - except Exception as e: - self._handle_portfolio_exceptions(e, requested_email, domain_org) - # If that first invite does not succeed take an early exit - return redirect(self.get_success_url()) - + member_of_a_different_org, member_of_this_org = get_org_membership(domain_org, requested_email, requested_user) try: + # determine portfolio of the domain (code currently is looking at requestor's portfolio) + # if requested_email/user is not member or invited member of this portfolio + # COMMENT: this code does not take into account multiple portfolios flag + # send portfolio invitation email + # create portfolio invitation + # create message to view + if ( + flag_is_active_for_user(requestor, "organization_feature") + and not flag_is_active_for_user(requestor, "multiple_portfolios") + and domain_org is not None + and requestor_can_update_portfolio + and not member_of_this_org + ): + send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) + PortfolioInvitation.objects.get_or_create( + email=requested_email, portfolio=domain_org, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + ) + messages.success(self.request, f"{requested_email} has been invited to the organization: {domain_org}") + if requested_user is None: self._handle_new_user_invitation(requested_email, requestor, member_of_a_different_org) else: self._handle_existing_user(requested_email, requestor, requested_user, member_of_a_different_org) except Exception as e: - self._handle_exceptions(e, requested_email) + handle_invitation_exceptions(self.request, e, requested_email) return redirect(self.get_success_url()) - def _get_requested_user(self, email): - """Retrieve a user by email or return None if the user doesn't exist.""" - try: - return User.objects.get(email=email) - except User.DoesNotExist: - return None - def _handle_new_user_invitation(self, email, requestor, member_of_different_org): """Handle invitation for a new user who does not exist in the system.""" send_domain_invitation_email( @@ -1265,57 +1222,6 @@ class DomainAddUserView(DomainFormBaseView): ) messages.success(self.request, f"Added user {email}.") - def _handle_exceptions(self, exception, email): - """Handle exceptions raised during the process.""" - if isinstance(exception, EmailSendingError): - logger.warning( - "Could not send email invitation to %s for domain %s (EmailSendingError)", - email, - self.object, - exc_info=True, - ) - messages.warning(self.request, "Could not send email invitation.") - elif isinstance(exception, OutsideOrgMemberError): - logger.warning( - "Could not send email. Can not invite member of a .gov organization to a different organization.", - self.object, - exc_info=True, - ) - messages.error( - self.request, - f"{email} is already a member of another .gov organization.", - ) - elif isinstance(exception, AlreadyDomainManagerError): - messages.warning(self.request, str(exception)) - elif isinstance(exception, AlreadyDomainInvitedError): - messages.warning(self.request, str(exception)) - elif isinstance(exception, MissingEmailError): - messages.error(self.request, str(exception)) - logger.error( - f"Can't send email to '{email}' on domain '{self.object}'. No email exists for the requestor.", - exc_info=True, - ) - elif isinstance(exception, IntegrityError): - messages.warning(self.request, f"{email} is already a manager for this domain") - else: - logger.warning("Could not send email invitation (Other Exception)", exc_info=True) - messages.warning(self.request, "Could not send email invitation.") - - def _handle_portfolio_exceptions(self, exception, email, portfolio): - """Handle exceptions raised during the process.""" - if isinstance(exception, EmailSendingError): - logger.warning("Could not send email invitation (EmailSendingError)", exc_info=True) - messages.warning(self.request, "Could not send email invitation.") - elif isinstance(exception, MissingEmailError): - messages.error(self.request, str(exception)) - logger.error( - f"Can't send email to '{email}' for portfolio '{portfolio}'. No email exists for the requestor.", - exc_info=True, - ) - else: - logger.warning("Could not send email invitation (Other Exception)", exc_info=True) - messages.warning(self.request, "Could not send email invitation.") - class DomainInvitationCancelView(SuccessMessageMixin, DomainInvitationPermissionCancelView): object: DomainInvitation diff --git a/src/registrar/views/utility/portfolio_helper.py b/src/registrar/views/utility/portfolio_helper.py new file mode 100644 index 000000000..6fa2d7e60 --- /dev/null +++ b/src/registrar/views/utility/portfolio_helper.py @@ -0,0 +1,83 @@ +from django.contrib import messages +from django.db import IntegrityError +from registrar.models.portfolio_invitation import PortfolioInvitation +from registrar.models.user import User +from registrar.models.user_portfolio_permission import UserPortfolioPermission +from registrar.utility.email import EmailSendingError +import logging + +from registrar.utility.errors import ( + AlreadyDomainInvitedError, + AlreadyDomainManagerError, + MissingEmailError, + OutsideOrgMemberError, +) + +logger = logging.getLogger(__name__) + + +def get_org_membership(requestor_org, requested_email, requested_user): + """ + Verifies if an email belongs to a different organization as a member or invited member. + Verifies if an email belongs to this organization as a member or invited member. + User does not belong to any org can be deduced from the tuple returned. + + Returns a tuple (member_of_a_different_org, member_of_this_org). + """ + + # COMMENT: this code does not take into account multiple portfolios flag + + # COMMENT: shouldn't this code be based on the organization of the domain, not the org + # of the requestor? requestor could have multiple portfolios + + # Check for existing permissions or invitations for the requested user + existing_org_permission = UserPortfolioPermission.objects.filter(user=requested_user).first() + existing_org_invitation = PortfolioInvitation.objects.filter(email=requested_email).first() + + # Determine membership in a different organization + member_of_a_different_org = (existing_org_permission and existing_org_permission.portfolio != requestor_org) or ( + existing_org_invitation and existing_org_invitation.portfolio != requestor_org + ) + + # Determine membership in the same organization + member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == requestor_org) or ( + existing_org_invitation and existing_org_invitation.portfolio == requestor_org + ) + + return member_of_a_different_org, member_of_this_org + + +def get_requested_user(email): + """Retrieve a user by email or return None if the user doesn't exist.""" + try: + return User.objects.get(email=email) + except User.DoesNotExist: + return None + + +def handle_invitation_exceptions(request, exception, email): + """Handle exceptions raised during the process.""" + if isinstance(exception, EmailSendingError): + logger.warning(str(exception), exc_info=True) + messages.error(request, str(exception)) + elif isinstance(exception, MissingEmailError): + messages.error(request, str(exception)) + logger.error(str(exception), exc_info=True) + elif isinstance(exception, OutsideOrgMemberError): + logger.warning( + "Could not send email. Can not invite member of a .gov organization to a different organization.", + exc_info=True, + ) + messages.error( + request, + f"{email} is already a member of another .gov organization.", + ) + elif isinstance(exception, AlreadyDomainManagerError): + messages.warning(request, str(exception)) + elif isinstance(exception, AlreadyDomainInvitedError): + messages.warning(request, str(exception)) + elif isinstance(exception, IntegrityError): + messages.warning(request, f"{email} is already a manager for this domain") + else: + logger.warning("Could not send email invitation (Other Exception)", exc_info=True) + messages.warning(request, "Could not send email invitation.") From 3d237ba0f5e3f677ab607551f3428375478cec0e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 Jan 2025 18:05:34 -0500 Subject: [PATCH 02/58] auto retrieve portfolio invitations on create --- src/registrar/admin.py | 31 +++++++++++++++++-- .../models/utility/portfolio_helper.py | 4 ++- src/registrar/views/domain.py | 5 ++- src/registrar/views/portfolios.py | 4 ++- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 183d251b4..0f0c76ee3 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1467,11 +1467,14 @@ class DomainInvitationAdmin(ListHeaderAdmin): and not member_of_this_org ): send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) - PortfolioInvitation.objects.get_or_create( + portfolio_invitation, _ = PortfolioInvitation.objects.get_or_create( email=requested_email, portfolio=domain_org, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], ) + if requested_user is not None: + portfolio_invitation.retrieve() + portfolio_invitation.save() messages.success(request, f"{requested_email} has been invited to the organization: {domain_org}") send_domain_invitation_email( @@ -1548,7 +1551,16 @@ class DomainInvitationAdmin(ListHeaderAdmin): change=False, obj=obj, ) - return super().response_add(request, obj, post_url_continue) + # Preserve all success messages + all_messages = [message for message in get_messages(request)] + + response = super().response_add(request, obj, post_url_continue) + + # Re-add all messages to the storage after `super().response_add` to preserve them + for message in all_messages: + messages.add_message(request, message.level, message.message) + + return response class PortfolioInvitationAdmin(ListHeaderAdmin): @@ -1612,6 +1624,8 @@ class PortfolioInvitationAdmin(ListHeaderAdmin): portfolio = obj.portfolio requested_email = obj.email requestor = request.user + # Look up a user with that email + requested_user = get_requested_user(requested_email) permission_exists = UserPortfolioPermission.objects.filter( user__email=requested_email, portfolio=portfolio, user__email__isnull=False @@ -1620,6 +1634,8 @@ class PortfolioInvitationAdmin(ListHeaderAdmin): if not permission_exists: # if permission does not exist for a user with requested_email, send email send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) + if requested_user is not None: + obj.retrieve() messages.success(request, f"{requested_email} has been invited.") else: messages.warning(request, "User is already a member of this portfolio.") @@ -1685,7 +1701,16 @@ class PortfolioInvitationAdmin(ListHeaderAdmin): change=False, obj=obj, ) - return super().response_add(request, obj, post_url_continue) + # Preserve all success messages + all_messages = [message for message in get_messages(request)] + + response = super().response_add(request, obj, post_url_continue) + + # Re-add all messages to the storage after `super().response_add` to preserve them + for message in all_messages: + messages.add_message(request, message.level, message.message) + + return response class DomainInformationResource(resources.ModelResource): diff --git a/src/registrar/models/utility/portfolio_helper.py b/src/registrar/models/utility/portfolio_helper.py index cde28e4bd..4ae282f21 100644 --- a/src/registrar/models/utility/portfolio_helper.py +++ b/src/registrar/models/utility/portfolio_helper.py @@ -153,7 +153,9 @@ def validate_user_portfolio_permission(user_portfolio_permission): "Based on current waffle flag settings, users cannot be assigned to multiple portfolios." ) - existing_invitations = PortfolioInvitation.objects.filter(email=user_portfolio_permission.user.email) + existing_invitations = PortfolioInvitation.objects.exclude( + portfolio=user_portfolio_permission.portfolio + ).filter(email=user_portfolio_permission.user.email) if existing_invitations.exists(): raise ValidationError( "This user is already assigned to a portfolio invitation. " diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 14e7dc933..b8464464e 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -1182,9 +1182,12 @@ class DomainAddUserView(DomainFormBaseView): and not member_of_this_org ): send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) - PortfolioInvitation.objects.get_or_create( + portfolio_invitation, _ = PortfolioInvitation.objects.get_or_create( email=requested_email, portfolio=domain_org, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] ) + if requested_user is not None: + portfolio_invitation.retrieve() + portfolio_invitation.save() messages.success(self.request, f"{requested_email} has been invited to the organization: {domain_org}") if requested_user is None: diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 751e28d85..60b30ad60 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -754,7 +754,9 @@ class PortfolioAddMemberView(PortfolioMembersPermissionView, FormMixin): try: if not requested_user or not permission_exists: send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) - form.save() + portfolio_invitation = form.save() + portfolio_invitation.retrieve() + portfolio_invitation.save() messages.success(self.request, f"{requested_email} has been invited.") else: if permission_exists: From 5e5626aaba19173aa9a7cfb7b9f43031bcb8e486 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:54:54 -0700 Subject: [PATCH 03/58] basic logic --- .../commands/create_federal_portfolio.py | 94 +++++++++++-- .../commands/patch_suborganizations.py | 129 ++++++++++++++++++ 2 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 src/registrar/management/commands/patch_suborganizations.py diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 9cf4d36ea..f3debba56 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -5,7 +5,7 @@ import logging from django.core.management import BaseCommand, CommandError from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User - +from django.db.models import F logger = logging.getLogger(__name__) @@ -99,17 +99,83 @@ class Command(BaseCommand): display_as_str=True, ) + # TODO - add post processing step to add suborg city, state, etc. + # This needs to be done after because of execution order. + # However, we do not need to necessarily prompt the user in this case. + def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" portfolio, created = self.create_portfolio(federal_agency) + suborganizations = Suborganization.objects.none() + domains = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) + domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( + status__in=[ + DomainRequest.DomainRequestStatus.STARTED, + DomainRequest.DomainRequestStatus.INELIGIBLE, + DomainRequest.DomainRequestStatus.REJECTED, + ] + ) if created: - self.create_suborganizations(portfolio, federal_agency) + suborganizations = self.create_suborganizations(portfolio, federal_agency) if parse_domains or both: - self.handle_portfolio_domains(portfolio, federal_agency) + domains = self.handle_portfolio_domains(portfolio, federal_agency, domains) if parse_requests or both: - self.handle_portfolio_requests(portfolio, federal_agency) + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency, domain_requests) + + # Post process steps + # Add suborg info to created or existing suborgs. Get the refreshed queryset for each. + self.post_process_suborganization(suborganizations.all(), domains.all(), domain_requests.all()) + + def post_process_suborganization(self, suborganizations, domains, requests): + # Exclude domains and requests where the org name is the same, + # and where we are missing some crucial information. + domains = domains.exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name") + ).in_bulk("organization_name") + + requests = requests.exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name") + ).in_bulk("organization_name") + + for suborg in suborganizations: + domain = domains.get(suborg.name, None) + request = requests.get(suborg.name, None) + + # PRIORITY: + # 1. Domain info + # 2. Domain request requested suborg fields + # 3. Domain request normal fields + city = None + if domain and domain.city: + city = normalize_string(domain.city, lowercase=False) + elif request and request.suborganization_city: + city = normalize_string(request.suborganization_city, lowercase=False) + elif request and request.city: + city = normalize_string(request.city, lowercase=False) + + state_territory = None + if domain and domain.state_territory: + state_territory = domain.state_territory + elif request and request.suborganization_state_territory: + state_territory = request.suborganization_state_territory + elif request and request.state_territory: + state_territory = request.state_territory + + if city: + suborg.city = city + + if suborg: + suborg.state_territory = state_territory + + Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) def create_portfolio(self, federal_agency): """Creates a portfolio if it doesn't presently exist. @@ -200,20 +266,13 @@ class Command(BaseCommand): ) else: TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") + return new_suborgs - def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): + def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_requests): """ Associate portfolio with domain requests for a federal agency. Updates all relevant domain request records. """ - invalid_states = [ - DomainRequest.DomainRequestStatus.STARTED, - DomainRequest.DomainRequestStatus.INELIGIBLE, - DomainRequest.DomainRequestStatus.REJECTED, - ] - domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( - status__in=invalid_states - ) if not domain_requests.exists(): message = f""" Portfolio '{portfolio}' not added to domain requests: no valid records found. @@ -238,12 +297,14 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): + # Return a fresh copy of the queryset + return domain_requests.all() + + def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_infos): """ Associate portfolio with domains for a federal agency. Updates all relevant domain information records. """ - domain_infos = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) if not domain_infos.exists(): message = f""" Portfolio '{portfolio}' not added to domains: no valid records found. @@ -263,3 +324,6 @@ class Command(BaseCommand): DomainInformation.objects.bulk_update(domain_infos, ["portfolio", "sub_organization"]) message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + + # Return a fresh copy of the queryset + return domain_infos.all() diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py new file mode 100644 index 000000000..197c6fe46 --- /dev/null +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -0,0 +1,129 @@ +@ -0,0 +1,123 @@ +import logging +from django.core.management import BaseCommand +from registrar.models import Suborganization, DomainRequest, DomainInformation +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" + + def handle(self, **kwargs): + + # Step 1: delete duplicates + # Find duplicates + duplicates = {} + all_suborgs = Suborganization.objects.all() + + for suborg in all_suborgs: + # Normalize name by removing extra spaces and converting to lowercase + normalized_name = " ".join(suborg.name.split()).lower() + + # First occurrence of this name + if normalized_name not in duplicates: + duplicates[normalized_name] = { + "keep": suborg, + "delete": [] + } + continue + + # Compare with our current best + current_best = duplicates[normalized_name]["keep"] + + # Check if all other fields match. + # If they don't, we should inspect this record manually. + fields_to_compare = ["portfolio", "city", "state_territory"] + fields_match = all( + getattr(suborg, field) == getattr(current_best, field) + for field in fields_to_compare + ) + if not fields_match: + logger.warning( + f"{TerminalColors.YELLOW}" + f"\nSkipping potential duplicate: {suborg.name} (id: {suborg.id})" + f"\nData mismatch with {current_best.name} (id: {current_best.id})" + f"{TerminalColors.ENDC}" + ) + continue + + # Determine if new suborg is better than current best. + # The fewest spaces and most capitals wins. + new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") + new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) + # TODO + # Split into words and count properly capitalized first letters + # new_proper_caps = sum( + # 1 for word in suborg.name.split() + # if word and word[0].isupper() + # ) + # current_proper_caps = sum( + # 1 for word in current_best.name.split() + # if word and word[0].isupper() + # ) + # new_has_better_caps = new_proper_caps > current_proper_caps + + if new_has_fewer_spaces or new_has_more_capitals: + # New suborg is better - demote the old one to the delete list + duplicates[normalized_name]["delete"].append(current_best) + duplicates[normalized_name]["keep"] = suborg + else: + # If it is not better, just delete the old one + duplicates[normalized_name]["delete"].append(suborg) + + # Filter out entries without duplicates + duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} + if not duplicates: + logger.info(f"No duplicate suborganizations found.") + return + + # Show preview of changes + preview = "The following duplicates will be removed:\n" + for data in duplicates.values(): + best = data.get("keep") + preview += f"\nKeeping: '{best.name}' (id: {best.id})" + + for duplicate in data.get("delete"): + preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" + preview += "\n" + + # Get confirmation and execute deletions + if TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message=preview, + prompt_title="Clean up duplicate suborganizations?", + verify_message="*** WARNING: This will delete suborganizations! ***" + ): + try: + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] + + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + ids_to_delete = [ + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ] + + # Bulk delete all duplicates + delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() + logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") + + except Exception as e: + logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + + # Step 2: Add city, state, etc info to existing suborganizations. + From 14b6382a2dfbb221d62fabaf9d823f61bcbebdc4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:08:36 -0700 Subject: [PATCH 04/58] Add import --- .../commands/create_federal_portfolio.py | 10 ++++------ .../management/commands/patch_suborganizations.py | 15 ++++++++++----- src/registrar/models/utility/generic_helper.py | 9 +++++++++ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index f3debba56..37bd4765d 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -7,6 +7,8 @@ from registrar.management.commands.utility.terminal_helper import TerminalColors from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User from django.db.models import F +from registrar.models.utility.generic_helper import normalize_string + logger = logging.getLogger(__name__) @@ -99,10 +101,6 @@ class Command(BaseCommand): display_as_str=True, ) - # TODO - add post processing step to add suborg city, state, etc. - # This needs to be done after because of execution order. - # However, we do not need to necessarily prompt the user in this case. - def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" @@ -126,9 +124,9 @@ class Command(BaseCommand): # Post process steps # Add suborg info to created or existing suborgs. Get the refreshed queryset for each. - self.post_process_suborganization(suborganizations.all(), domains.all(), domain_requests.all()) + self.post_process_suborganization_fields(suborganizations.all(), domains.all(), domain_requests.all()) - def post_process_suborganization(self, suborganizations, domains, requests): + def post_process_suborganization_fields(self, suborganizations, domains, requests): # Exclude domains and requests where the org name is the same, # and where we are missing some crucial information. domains = domains.exclude( diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 197c6fe46..88ad611da 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -12,12 +12,19 @@ class Command(BaseCommand): help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" def handle(self, **kwargs): - + # Maybe we should just do these manually? + extra_records_to_delete = [ + "Assistant Secretary for Preparedness and Response Office of the Secretary", + "US Geological Survey", + "USDA/OC", + ] # Step 1: delete duplicates + self.delete_suborganization_duplicates() + + def delete_suborganization_duplicates(self, extra_records_to_delete): # Find duplicates duplicates = {} all_suborgs = Suborganization.objects.all() - for suborg in all_suborgs: # Normalize name by removing extra spaces and converting to lowercase normalized_name = " ".join(suborg.name.split()).lower() @@ -124,6 +131,4 @@ class Command(BaseCommand): except Exception as e: logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") - - # Step 2: Add city, state, etc info to existing suborganizations. - + diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 5e425f5a3..84dc28db1 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -343,3 +343,12 @@ def value_of_attribute(obj, attribute_name: str): if callable(value): value = value() return value + +def normalize_string(string_to_normalize, lowercase=True): + """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" + if not isinstance(string_to_normalize, str): + logger.error(f"normalize_string => {string_to_normalize} is not type str.") + return string_to_normalize + + new_string = " ".join(string_to_normalize.split()) + return new_string.lower() if lowercase else new_string From ae2e4c59995ec954403f9941be5092a87c69171d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:44:18 -0700 Subject: [PATCH 05/58] Ducttape --- .../commands/patch_suborganizations.py | 103 ++++++++++++------ 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 88ad611da..07fb94f3b 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -3,6 +3,7 @@ import logging from django.core.management import BaseCommand from registrar.models import Suborganization, DomainRequest, DomainInformation from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper +from registrar.models.utility.generic_helper import normalize_string logger = logging.getLogger(__name__) @@ -12,18 +13,58 @@ class Command(BaseCommand): help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" def handle(self, **kwargs): - # Maybe we should just do these manually? - extra_records_to_delete = [ + manual_records = [ "Assistant Secretary for Preparedness and Response Office of the Secretary", "US Geological Survey", "USDA/OC", ] - # Step 1: delete duplicates - self.delete_suborganization_duplicates() - - def delete_suborganization_duplicates(self, extra_records_to_delete): - # Find duplicates duplicates = {} + for record in Suborganization.objects.filter(name__in=manual_records): + if record.name: + norm_name = normalize_string(record.name) + duplicates[norm_name] = { + "keep": None, + "delete": [record] + } + + records_to_delete.update(self.handle_suborganization_duplicates()) + + # Get confirmation and execute deletions + if TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + prompt_message=preview, + prompt_title="Clean up duplicate suborganizations?", + verify_message="*** WARNING: This will delete suborganizations! ***" + ): + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] + + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + records_to_delete = set( + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ) + try: + delete_count, _ = Suborganization.objects.filter(id__in=records_to_delete).delete() + logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") + except Exception as e: + logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + + + def handle_suborganization_duplicates(self, duplicates): + # Find duplicates all_suborgs = Suborganization.objects.all() for suborg in all_suborgs: # Normalize name by removing extra spaces and converting to lowercase @@ -103,32 +144,26 @@ class Command(BaseCommand): prompt_title="Clean up duplicate suborganizations?", verify_message="*** WARNING: This will delete suborganizations! ***" ): - try: - # Update all references to point to the right suborg before deletion - for record in duplicates.values(): - best_record = record.get("keep") - delete_ids = [dupe.id for dupe in record.get("delete")] - - # Update domain requests - DomainRequest.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - # Update domain information - DomainInformation.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - ids_to_delete = [ - dupe.id - for data in duplicates.values() - for dupe in data["delete"] - ] + # Update all references to point to the right suborg before deletion + for record in duplicates.values(): + best_record = record.get("keep") + delete_ids = [dupe.id for dupe in record.get("delete")] - # Bulk delete all duplicates - delete_count, _ = Suborganization.objects.filter(id__in=ids_to_delete).delete() - logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") - - except Exception as e: - logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") + # Update domain requests + DomainRequest.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + + # Update domain information + DomainInformation.objects.filter( + sub_organization_id__in=delete_ids + ).update(sub_organization=best_record) + records_to_delete = set( + dupe.id + for data in duplicates.values() + for dupe in data["delete"] + ) + return records_to_delete + else: + return set() From b048ff96de5565042e126ffd64cbdc3750a5face Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 8 Jan 2025 18:26:56 -0500 Subject: [PATCH 06/58] handle multiple domains in email notifications --- src/registrar/admin.py | 3 +- .../templates/emails/domain_invitation.txt | 41 +++++---- .../emails/domain_invitation_subject.txt | 2 +- src/registrar/utility/email_invitations.py | 50 +++++++---- src/registrar/views/domain.py | 5 +- src/registrar/views/portfolios.py | 85 ++++++++++++------- 6 files changed, 119 insertions(+), 67 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 0f0c76ee3..06731db38 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1480,8 +1480,9 @@ class DomainInvitationAdmin(ListHeaderAdmin): send_domain_invitation_email( email=requested_email, requestor=requestor, - domain=domain, + domains=domain, is_member_of_different_org=member_of_a_different_org, + requested_user=requested_user ) if requested_user is not None: # Domain Invitation creation for an existing User diff --git a/src/registrar/templates/emails/domain_invitation.txt b/src/registrar/templates/emails/domain_invitation.txt index 068040205..4959f7c23 100644 --- a/src/registrar/templates/emails/domain_invitation.txt +++ b/src/registrar/templates/emails/domain_invitation.txt @@ -1,36 +1,46 @@ {% autoescape off %}{# In a text file, we don't want to have HTML entities escaped #} -Hi. +{% if requested_user and requested_user.first_name %} +Hi, {{ requested_user.first_name }}. +{% else %} +Hi, +{% endif %} -{{ requestor_email }} has added you as a manager on {{ domain.name }}. +{{ requestor_email }} has invited you to manage: +{% for domain in domains %} +{{ domain.name }} +{% endfor %} -You can manage this domain on the .gov registrar . +To manage domain information, visit the .gov registrar . ---------------------------------------------------------------- +{% if not requested_user %} YOU NEED A LOGIN.GOV ACCOUNT -You’ll need a Login.gov account to manage your .gov domain. Login.gov provides -a simple and secure process for signing in to many government services with one -account. +You’ll need a Login.gov account to access the .gov registrar. That account needs to be +associated with the following email address: {{ invitee_email_address }} -If you don’t already have one, follow these steps to create your -Login.gov account . +Login.gov provides a simple and secure process for signing in to many government +services with one account. If you don’t already have one, follow these steps to create +your Login.gov account . +{% endif %} DOMAIN MANAGEMENT -As a .gov domain manager, you can add or update information about your domain. -You’ll also serve as a contact for your .gov domain. Please keep your contact -information updated. +As a .gov domain manager, you can add or update information like name servers. You’ll +also serve as a contact for the domains you manage. Please keep your contact +information updated. Learn more about domain management . SOMETHING WRONG? -If you’re not affiliated with {{ domain.name }} or think you received this -message in error, reply to this email. +If you’re not affiliated with the .gov domains mentioned in this invitation or think you +received this message in error, reply to this email. THANK YOU -.Gov helps the public identify official, trusted information. Thank you for using a .gov domain. +.Gov helps the public identify official, trusted information. Thank you for using a .gov +domain. ---------------------------------------------------------------- @@ -38,5 +48,6 @@ The .gov team Contact us: Learn about .gov -The .gov registry is a part of the Cybersecurity and Infrastructure Security Agency (CISA) +The .gov registry is a part of the Cybersecurity and Infrastructure Security Agency +(CISA) {% endautoescape %} diff --git a/src/registrar/templates/emails/domain_invitation_subject.txt b/src/registrar/templates/emails/domain_invitation_subject.txt index 319b80176..9663346d0 100644 --- a/src/registrar/templates/emails/domain_invitation_subject.txt +++ b/src/registrar/templates/emails/domain_invitation_subject.txt @@ -1 +1 @@ -You’ve been added to a .gov domain \ No newline at end of file +You've been invited to manage {% if domains|length > 1 %}.gov domains{% else %}{{ domains.0.name }}{% endif %} \ No newline at end of file diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 9455c5927..630da7ce5 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -1,5 +1,7 @@ from django.conf import settings from registrar.models import DomainInvitation +from registrar.models.domain import Domain +from registrar.models.user import User from registrar.utility.errors import ( AlreadyDomainInvitedError, AlreadyDomainManagerError, @@ -13,7 +15,7 @@ import logging logger = logging.getLogger(__name__) -def send_domain_invitation_email(email: str, requestor, domain, is_member_of_different_org): +def send_domain_invitation_email(email: str, requestor, domains: Domain | list[Domain], is_member_of_different_org, requested_user=None): """ Sends a domain invitation email to the specified address. @@ -22,8 +24,9 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif Args: email (str): Email address of the recipient. requestor (User): The user initiating the invitation. - domain (Domain): The domain object for which the invitation is being sent. + domains (Domain or list of Domain): The domain objects for which the invitation is being sent. is_member_of_different_org (bool): if an email belongs to a different org + requested_user (User | None): The recipient if the email belongs to a user in the registrar Raises: MissingEmailError: If the requestor has no email associated with their account. @@ -32,13 +35,19 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif OutsideOrgMemberError: If the requested_user is part of a different organization. EmailSendingError: If there is an error while sending the email. """ + print('send_domain_invitation_email') + # Normalize domains + if isinstance(domains, Domain): + domains = [domains] + # Default email address for staff requestor_email = settings.DEFAULT_FROM_EMAIL # Check if the requestor is staff and has an email if not requestor.is_staff: if not requestor.email or requestor.email.strip() == "": - raise MissingEmailError(email=email, domain=domain) + domain_names = ", ".join([domain.name for domain in domains]) + raise MissingEmailError(email=email, domain=domain_names) else: requestor_email = requestor.email @@ -51,18 +60,19 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif ): raise OutsideOrgMemberError - # Check for an existing invitation - try: - invite = DomainInvitation.objects.get(email=email, domain=domain) - if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: - raise AlreadyDomainManagerError(email) - elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: - invite.update_cancellation_status() - invite.save() - else: - raise AlreadyDomainInvitedError(email) - except DomainInvitation.DoesNotExist: - pass + # Check for an existing invitation for each domain + for domain in domains: + try: + invite = DomainInvitation.objects.get(email=email, domain=domain) + if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: + raise AlreadyDomainManagerError(email) + elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: + invite.update_cancellation_status() + invite.save() + else: + raise AlreadyDomainInvitedError(email) + except DomainInvitation.DoesNotExist: + pass # Send the email try: @@ -71,12 +81,18 @@ def send_domain_invitation_email(email: str, requestor, domain, is_member_of_dif "emails/domain_invitation_subject.txt", to_address=email, context={ - "domain": domain, + "domains": domains, "requestor_email": requestor_email, + "invitee_email_address": email, + "requested_user": requested_user, }, ) except EmailSendingError as err: - raise EmailSendingError(f"Could not send email invitation to {email} for domain {domain}.") from err + print('point of failure test') + domain_names = ", ".join([domain.name for domain in domains]) + raise EmailSendingError( + f"Could not send email invitation to {email} for domains: {domain_names}" + ) from err def send_portfolio_invitation_email(email: str, requestor, portfolio): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b8464464e..f7938a301 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -1204,7 +1204,7 @@ class DomainAddUserView(DomainFormBaseView): send_domain_invitation_email( email=email, requestor=requestor, - domain=self.object, + domains=self.object, is_member_of_different_org=member_of_different_org, ) DomainInvitation.objects.get_or_create(email=email, domain=self.object) @@ -1215,8 +1215,9 @@ class DomainAddUserView(DomainFormBaseView): send_domain_invitation_email( email=email, requestor=requestor, - domain=self.object, + domains=self.object, is_member_of_different_org=member_of_different_org, + requested_user=requested_user, ) UserDomainRole.objects.create( user=requested_user, diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 60b30ad60..0cca1280c 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -8,13 +8,14 @@ from django.utils.safestring import mark_safe from django.contrib import messages from registrar.forms import portfolio as portfolioForms from registrar.models import Portfolio, User +from registrar.models.domain import Domain from registrar.models.domain_invitation import DomainInvitation from registrar.models.portfolio_invitation import PortfolioInvitation from registrar.models.user_domain_role import UserDomainRole from registrar.models.user_portfolio_permission import UserPortfolioPermission from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from registrar.utility.email import EmailSendingError -from registrar.utility.email_invitations import send_portfolio_invitation_email +from registrar.utility.email_invitations import send_domain_invitation_email, send_portfolio_invitation_email from registrar.utility.errors import MissingEmailError from registrar.utility.enums import DefaultUserValues from registrar.views.utility.mixins import PortfolioMemberPermission @@ -33,6 +34,8 @@ from django.views.generic import View from django.views.generic.edit import FormMixin from django.db import IntegrityError +from registrar.views.utility.portfolio_helper import get_org_membership + logger = logging.getLogger(__name__) @@ -237,6 +240,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V removed_domains = request.POST.get("removed_domains") portfolio_permission = get_object_or_404(UserPortfolioPermission, pk=pk) member = portfolio_permission.user + portfolio = portfolio_permission.portfolio added_domain_ids = self._parse_domain_ids(added_domains, "added domains") if added_domain_ids is None: @@ -248,7 +252,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V if added_domain_ids or removed_domain_ids: try: - self._process_added_domains(added_domain_ids, member) + self._process_added_domains(added_domain_ids, member, request.user, portfolio) self._process_removed_domains(removed_domain_ids, member) messages.success(request, "The domain assignment changes have been saved.") return redirect(reverse("member-domains", kwargs={"pk": pk})) @@ -263,7 +267,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V except Exception as e: messages.error( request, - "An unexpected error occurred: {str(e)}. If the issue persists, " + f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) logger.error(f"An unexpected error occurred: {str(e)}") @@ -287,16 +291,26 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V logger.error(f"Invalid data for {domain_type}") return None - def _process_added_domains(self, added_domain_ids, member): + def _process_added_domains(self, added_domain_ids, member, requestor, portfolio): """ Processes added domains by bulk creating UserDomainRole instances. """ if added_domain_ids: + print('_process_added_domains') + added_domains = Domain.objects.filter(id__in=added_domain_ids) + member_of_a_different_org, _ = get_org_membership(portfolio, member.email, member) + send_domain_invitation_email( + email=member.email, + requestor=requestor, + domains=added_domains, + is_member_of_different_org=member_of_a_different_org, + requested_user=member, + ) # Bulk create UserDomainRole instances for added domains UserDomainRole.objects.bulk_create( [ - UserDomainRole(domain_id=domain_id, user=member, role=UserDomainRole.Roles.MANAGER) - for domain_id in added_domain_ids + UserDomainRole(domain=domain, user=member, role=UserDomainRole.Roles.MANAGER) + for domain in added_domains ], ignore_conflicts=True, # Avoid duplicate entries ) @@ -443,6 +457,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission removed_domains = request.POST.get("removed_domains") portfolio_invitation = get_object_or_404(PortfolioInvitation, pk=pk) email = portfolio_invitation.email + portfolio = portfolio_invitation.portfolio added_domain_ids = self._parse_domain_ids(added_domains, "added domains") if added_domain_ids is None: @@ -454,7 +469,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission if added_domain_ids or removed_domain_ids: try: - self._process_added_domains(added_domain_ids, email) + self._process_added_domains(added_domain_ids, email, request.user, portfolio) self._process_removed_domains(removed_domain_ids, email) messages.success(request, "The domain assignment changes have been saved.") return redirect(reverse("invitedmember-domains", kwargs={"pk": pk})) @@ -469,7 +484,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission except Exception as e: messages.error( request, - "An unexpected error occurred: {str(e)}. If the issue persists, " + f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) logger.error(f"An unexpected error occurred: {str(e)}.") @@ -493,34 +508,41 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission logger.error(f"Invalid data for {domain_type}.") return None - def _process_added_domains(self, added_domain_ids, email): + def _process_added_domains(self, added_domain_ids, email, requestor, portfolio): """ Processes added domain invitations by updating existing invitations or creating new ones. """ - if not added_domain_ids: - return - - # Update existing invitations from CANCELED to INVITED - existing_invitations = DomainInvitation.objects.filter(domain_id__in=added_domain_ids, email=email) - existing_invitations.update(status=DomainInvitation.DomainInvitationStatus.INVITED) - - # Determine which domains need new invitations - existing_domain_ids = existing_invitations.values_list("domain_id", flat=True) - new_domain_ids = set(added_domain_ids) - set(existing_domain_ids) - - # Bulk create new invitations - DomainInvitation.objects.bulk_create( - [ - DomainInvitation( - domain_id=domain_id, + if added_domain_ids: + added_domains = Domain.objects.filter(id__in=added_domain_ids) + member_of_a_different_org, _ = get_org_membership(portfolio, email, None) + send_domain_invitation_email( email=email, - status=DomainInvitation.DomainInvitationStatus.INVITED, + requestor=requestor, + domains=added_domains, + is_member_of_different_org=member_of_a_different_org, ) - for domain_id in new_domain_ids - ] - ) + # Update existing invitations from CANCELED to INVITED + existing_invitations = DomainInvitation.objects.filter(domain__in=added_domains, email=email) + existing_invitations.update(status=DomainInvitation.DomainInvitationStatus.INVITED) + + # Determine which domains need new invitations + existing_domain_ids = existing_invitations.values_list("domain_id", flat=True) + new_domain_ids = set(added_domain_ids) - set(existing_domain_ids) + + # Bulk create new invitations + DomainInvitation.objects.bulk_create( + [ + DomainInvitation( + domain_id=domain_id, + email=email, + status=DomainInvitation.DomainInvitationStatus.INVITED, + ) + for domain_id in new_domain_ids + ] + ) + def _process_removed_domains(self, removed_domain_ids, email): """ Processes removed domain invitations by updating their status to CANCELED. @@ -755,8 +777,9 @@ class PortfolioAddMemberView(PortfolioMembersPermissionView, FormMixin): if not requested_user or not permission_exists: send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) portfolio_invitation = form.save() - portfolio_invitation.retrieve() - portfolio_invitation.save() + if requested_user is not None: + portfolio_invitation.retrieve() + portfolio_invitation.save() messages.success(self.request, f"{requested_email} has been invited.") else: if permission_exists: From 9a8b236d5d6fdea79e200a0aa6e4067a7a5e5d07 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 07:24:30 -0500 Subject: [PATCH 07/58] refactored admin.py and renamed helper file --- src/registrar/admin.py | 208 ++++++------------ src/registrar/views/domain.py | 2 +- src/registrar/views/portfolios.py | 2 +- ...rtfolio_helper.py => invitation_helper.py} | 3 + 4 files changed, 78 insertions(+), 137 deletions(-) rename src/registrar/views/utility/{portfolio_helper.py => invitation_helper.py} (93%) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 06731db38..e6f0f7929 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -29,7 +29,7 @@ from django_fsm import get_available_FIELD_transitions, FSMField from registrar.models import DomainInformation, Portfolio, UserPortfolioPermission, DomainInvitation from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from registrar.utility.email_invitations import send_domain_invitation_email, send_portfolio_invitation_email -from registrar.views.utility.portfolio_helper import ( +from registrar.views.utility.invitation_helper import ( get_org_membership, get_requested_user, handle_invitation_exceptions, @@ -1393,8 +1393,78 @@ class UserDomainRoleAdmin(ListHeaderAdmin, ImportExportModelAdmin): return super().changeform_view(request, object_id, form_url, extra_context=extra_context) +class BaseInvitationAdmin(ListHeaderAdmin): + """Base class for admin classes which will customize save_model and send email invitations + on model adds, and require custom handling of forms and form errors.""" -class DomainInvitationAdmin(ListHeaderAdmin): + def response_add(self, request, obj, post_url_continue=None): + """ + Override response_add to handle rendering when exceptions are raised during add model. + + Normal flow on successful save_model on add is to redirect to changelist_view. + If there are errors, flow is modified to instead render change form. + """ + # store current messages from request so that they are preserved throughout the method + storage = get_messages(request) + # Check if there are any error or warning messages in the `messages` framework + has_errors = any(message.level_tag in ["error", "warning"] for message in storage) + + if has_errors: + # Re-render the change form if there are errors or warnings + # Prepare context for rendering the change form + + # Get the model form + ModelForm = self.get_form(request, obj=obj) + form = ModelForm(instance=obj) + + # Create an AdminForm instance + admin_form = AdminForm( + form, + list(self.get_fieldsets(request, obj)), + self.get_prepopulated_fields(request, obj), + self.get_readonly_fields(request, obj), + model_admin=self, + ) + media = self.media + form.media + + opts = obj._meta + change_form_context = { + **self.admin_site.each_context(request), # Add admin context + "title": f"Add {opts.verbose_name}", + "opts": opts, + "original": obj, + "save_as": self.save_as, + "has_change_permission": self.has_change_permission(request, obj), + "add": True, # Indicate this is an "Add" form + "change": False, # Indicate this is not a "Change" form + "is_popup": False, + "inline_admin_formsets": [], + "save_on_top": self.save_on_top, + "show_delete": self.has_delete_permission(request, obj), + "obj": obj, + "adminform": admin_form, # Pass the AdminForm instance + "media": media, + "errors": None, + } + return self.render_change_form( + request, + context=change_form_context, + add=True, + change=False, + obj=obj, + ) + + response = super().response_add(request, obj, post_url_continue) + + # Re-add all messages from storage after `super().response_add` + # as super().response_add resets the success messages in request + for message in storage: + messages.add_message(request, message.level, message.message) + + return response + + +class DomainInvitationAdmin(BaseInvitationAdmin): """Custom domain invitation admin class.""" class Meta: @@ -1497,74 +1567,8 @@ class DomainInvitationAdmin(ListHeaderAdmin): # Call the parent save method to save the object super().save_model(request, obj, form, change) - def response_add(self, request, obj, post_url_continue=None): - """ - Override response_add to handle rendering when exceptions are raised during add model. - Normal flow on successful save_model on add is to redirect to changelist_view. - If there are errors, flow is modified to instead render change form. - """ - # Check if there are any error or warning messages in the `messages` framework - storage = get_messages(request) - has_errors = any(message.level_tag in ["error", "warning"] for message in storage) - - if has_errors: - # Re-render the change form if there are errors or warnings - # Prepare context for rendering the change form - - # Get the model form - ModelForm = self.get_form(request, obj=obj) - form = ModelForm(instance=obj) - - # Create an AdminForm instance - admin_form = AdminForm( - form, - list(self.get_fieldsets(request, obj)), - self.get_prepopulated_fields(request, obj), - self.get_readonly_fields(request, obj), - model_admin=self, - ) - media = self.media + form.media - - opts = obj._meta - change_form_context = { - **self.admin_site.each_context(request), # Add admin context - "title": f"Add {opts.verbose_name}", - "opts": opts, - "original": obj, - "save_as": self.save_as, - "has_change_permission": self.has_change_permission(request, obj), - "add": True, # Indicate this is an "Add" form - "change": False, # Indicate this is not a "Change" form - "is_popup": False, - "inline_admin_formsets": [], - "save_on_top": self.save_on_top, - "show_delete": self.has_delete_permission(request, obj), - "obj": obj, - "adminform": admin_form, # Pass the AdminForm instance - "media": media, - "errors": None, - } - return self.render_change_form( - request, - context=change_form_context, - add=True, - change=False, - obj=obj, - ) - # Preserve all success messages - all_messages = [message for message in get_messages(request)] - - response = super().response_add(request, obj, post_url_continue) - - # Re-add all messages to the storage after `super().response_add` to preserve them - for message in all_messages: - messages.add_message(request, message.level, message.message) - - return response - - -class PortfolioInvitationAdmin(ListHeaderAdmin): +class PortfolioInvitationAdmin(BaseInvitationAdmin): """Custom portfolio invitation admin class.""" form = PortfolioInvitationAdminForm @@ -1647,72 +1651,6 @@ class PortfolioInvitationAdmin(ListHeaderAdmin): # Call the parent save method to save the object super().save_model(request, obj, form, change) - def response_add(self, request, obj, post_url_continue=None): - """ - Override response_add to handle rendering when exceptions are raised during add model. - - Normal flow on successful save_model on add is to redirect to changelist_view. - If there are errors, flow is modified to instead render change form. - """ - # Check if there are any error or warning messages in the `messages` framework - storage = get_messages(request) - has_errors = any(message.level_tag in ["error", "warning"] for message in storage) - - if has_errors: - # Re-render the change form if there are errors or warnings - # Prepare context for rendering the change form - - # Get the model form - ModelForm = self.get_form(request, obj=obj) - form = ModelForm(instance=obj) - - # Create an AdminForm instance - admin_form = AdminForm( - form, - list(self.get_fieldsets(request, obj)), - self.get_prepopulated_fields(request, obj), - self.get_readonly_fields(request, obj), - model_admin=self, - ) - media = self.media + form.media - - opts = obj._meta - change_form_context = { - **self.admin_site.each_context(request), # Add admin context - "title": f"Add {opts.verbose_name}", - "opts": opts, - "original": obj, - "save_as": self.save_as, - "has_change_permission": self.has_change_permission(request, obj), - "add": True, # Indicate this is an "Add" form - "change": False, # Indicate this is not a "Change" form - "is_popup": False, - "inline_admin_formsets": [], - "save_on_top": self.save_on_top, - "show_delete": self.has_delete_permission(request, obj), - "obj": obj, - "adminform": admin_form, # Pass the AdminForm instance - "media": media, - "errors": None, - } - return self.render_change_form( - request, - context=change_form_context, - add=True, - change=False, - obj=obj, - ) - # Preserve all success messages - all_messages = [message for message in get_messages(request)] - - response = super().response_add(request, obj, post_url_continue) - - # Re-add all messages to the storage after `super().response_add` to preserve them - for message in all_messages: - messages.add_message(request, message.level, message.message) - - return response - class DomainInformationResource(resources.ModelResource): """defines how each field in the referenced model should be mapped to the corresponding fields in the diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index f7938a301..2e4f375eb 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -42,7 +42,7 @@ from registrar.utility.errors import ( from registrar.models.utility.contact_error import ContactError from registrar.views.utility.permission_views import UserDomainRolePermissionDeleteView from registrar.utility.waffle import flag_is_active_for_user -from registrar.views.utility.portfolio_helper import ( +from registrar.views.utility.invitation_helper import ( get_org_membership, get_requested_user, handle_invitation_exceptions, diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 0cca1280c..60a96bba6 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -34,7 +34,7 @@ from django.views.generic import View from django.views.generic.edit import FormMixin from django.db import IntegrityError -from registrar.views.utility.portfolio_helper import get_org_membership +from registrar.views.utility.invitation_helper import get_org_membership logger = logging.getLogger(__name__) diff --git a/src/registrar/views/utility/portfolio_helper.py b/src/registrar/views/utility/invitation_helper.py similarity index 93% rename from src/registrar/views/utility/portfolio_helper.py rename to src/registrar/views/utility/invitation_helper.py index 6fa2d7e60..8c3b1bff7 100644 --- a/src/registrar/views/utility/portfolio_helper.py +++ b/src/registrar/views/utility/invitation_helper.py @@ -15,6 +15,9 @@ from registrar.utility.errors import ( logger = logging.getLogger(__name__) +# These methods are used by multiple views which share similar logic and function +# when creating invitations and sending associated emails. These can be reused in +# any view, and were initially developed for domain.py, portfolios.py and admin.py def get_org_membership(requestor_org, requested_email, requested_user): """ From 94ed2c20b99141431535f61d92d66501e8aab498 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 07:33:32 -0500 Subject: [PATCH 08/58] code cleanup - removed unnecessary note --- src/registrar/models/domain_invitation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/registrar/models/domain_invitation.py b/src/registrar/models/domain_invitation.py index e2aede696..28089dcb5 100644 --- a/src/registrar/models/domain_invitation.py +++ b/src/registrar/models/domain_invitation.py @@ -56,8 +56,6 @@ class DomainInvitation(TimeStampedModel): Raises: RuntimeError if no matching user can be found. """ - # NOTE: this is currently not accounting for scenario when User.objects.get matches - # multiple user accounts with the same email address # get a user with this email address User = get_user_model() From b9626e2b9afb7a6275c81fca560faad6d2724645 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 07:42:17 -0500 Subject: [PATCH 09/58] removed print statements --- src/registrar/utility/email_invitations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 630da7ce5..55a0bdf14 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -35,7 +35,6 @@ def send_domain_invitation_email(email: str, requestor, domains: Domain | list[D OutsideOrgMemberError: If the requested_user is part of a different organization. EmailSendingError: If there is an error while sending the email. """ - print('send_domain_invitation_email') # Normalize domains if isinstance(domains, Domain): domains = [domains] @@ -88,7 +87,6 @@ def send_domain_invitation_email(email: str, requestor, domains: Domain | list[D }, ) except EmailSendingError as err: - print('point of failure test') domain_names = ", ".join([domain.name for domain in domains]) raise EmailSendingError( f"Could not send email invitation to {email} for domains: {domain_names}" From e06a63e3217f4015d1afb94830c777ad4e9a14c6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 07:47:17 -0500 Subject: [PATCH 10/58] updated comments related to multiple portfolio flags --- src/registrar/views/domain.py | 3 ++- src/registrar/views/utility/invitation_helper.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 2e4f375eb..b16e3681d 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -1168,9 +1168,10 @@ class DomainAddUserView(DomainFormBaseView): member_of_a_different_org, member_of_this_org = get_org_membership(domain_org, requested_email, requested_user) try: + # COMMENT: this code does not take into account multiple portfolios flag being set to TRUE + # determine portfolio of the domain (code currently is looking at requestor's portfolio) # if requested_email/user is not member or invited member of this portfolio - # COMMENT: this code does not take into account multiple portfolios flag # send portfolio invitation email # create portfolio invitation # create message to view diff --git a/src/registrar/views/utility/invitation_helper.py b/src/registrar/views/utility/invitation_helper.py index 8c3b1bff7..2cdde6a5e 100644 --- a/src/registrar/views/utility/invitation_helper.py +++ b/src/registrar/views/utility/invitation_helper.py @@ -28,7 +28,7 @@ def get_org_membership(requestor_org, requested_email, requested_user): Returns a tuple (member_of_a_different_org, member_of_this_org). """ - # COMMENT: this code does not take into account multiple portfolios flag + # COMMENT: this code does not take into account when multiple portfolios flag is set to true # COMMENT: shouldn't this code be based on the organization of the domain, not the org # of the requestor? requestor could have multiple portfolios From 7e2930f15dbae6cace514972f78b0bb4d1135d31 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 08:36:44 -0500 Subject: [PATCH 11/58] formatted for code readability --- src/registrar/admin.py | 5 +- src/registrar/utility/email_invitations.py | 90 +++++++++++-------- src/registrar/views/domain.py | 1 + src/registrar/views/portfolios.py | 16 ++-- .../views/utility/invitation_helper.py | 1 + 5 files changed, 69 insertions(+), 44 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index e6f0f7929..cdcc0400e 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1393,6 +1393,7 @@ class UserDomainRoleAdmin(ListHeaderAdmin, ImportExportModelAdmin): return super().changeform_view(request, object_id, form_url, extra_context=extra_context) + class BaseInvitationAdmin(ListHeaderAdmin): """Base class for admin classes which will customize save_model and send email invitations on model adds, and require custom handling of forms and form errors.""" @@ -1542,6 +1543,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): portfolio=domain_org, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], ) + # if user exists for email, immediately retrieve portfolio invitation upon creation if requested_user is not None: portfolio_invitation.retrieve() portfolio_invitation.save() @@ -1552,7 +1554,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): requestor=requestor, domains=domain, is_member_of_different_org=member_of_a_different_org, - requested_user=requested_user + requested_user=requested_user, ) if requested_user is not None: # Domain Invitation creation for an existing User @@ -1639,6 +1641,7 @@ class PortfolioInvitationAdmin(BaseInvitationAdmin): if not permission_exists: # if permission does not exist for a user with requested_email, send email send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) + # if user exists for email, immediately retrieve portfolio invitation upon creation if requested_user is not None: obj.retrieve() messages.success(request, f"{requested_email} has been invited.") diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 55a0bdf14..1491b65a5 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -1,7 +1,6 @@ from django.conf import settings from registrar.models import DomainInvitation from registrar.models.domain import Domain -from registrar.models.user import User from registrar.utility.errors import ( AlreadyDomainInvitedError, AlreadyDomainManagerError, @@ -15,12 +14,12 @@ import logging logger = logging.getLogger(__name__) -def send_domain_invitation_email(email: str, requestor, domains: Domain | list[Domain], is_member_of_different_org, requested_user=None): +def send_domain_invitation_email( + email: str, requestor, domains: Domain | list[Domain], is_member_of_different_org, requested_user=None +): """ Sends a domain invitation email to the specified address. - Raises exceptions for validation or email-sending issues. - Args: email (str): Email address of the recipient. requestor (User): The user initiating the invitation. @@ -35,45 +34,66 @@ def send_domain_invitation_email(email: str, requestor, domains: Domain | list[D OutsideOrgMemberError: If the requested_user is part of a different organization. EmailSendingError: If there is an error while sending the email. """ - # Normalize domains - if isinstance(domains, Domain): - domains = [domains] + domains = normalize_domains(domains) + requestor_email = get_requestor_email(requestor, domains) - # Default email address for staff - requestor_email = settings.DEFAULT_FROM_EMAIL + validate_invitation(email, domains, requestor, is_member_of_different_org) - # Check if the requestor is staff and has an email - if not requestor.is_staff: - if not requestor.email or requestor.email.strip() == "": - domain_names = ", ".join([domain.name for domain in domains]) - raise MissingEmailError(email=email, domain=domain_names) - else: - requestor_email = requestor.email + send_invitation_email(email, requestor_email, domains, requested_user) - # Check if the recipient is part of a different organization - # COMMENT: this does not account for multiple_portfolios flag being active + +def normalize_domains(domains): + """Ensures domains is always a list.""" + return [domains] if isinstance(domains, Domain) else domains + + +def get_requestor_email(requestor, domains): + """Get the requestor's email or raise an error if it's missing.""" + if requestor.is_staff: + return settings.DEFAULT_FROM_EMAIL + + if not requestor.email or requestor.email.strip() == "": + domain_names = ", ".join([domain.name for domain in domains]) + raise MissingEmailError(email=requestor.email, domain=domain_names) + + return requestor.email + + +def validate_invitation(email, domains, requestor, is_member_of_different_org): + """Validate the invitation conditions.""" + check_outside_org_membership(email, requestor, is_member_of_different_org) + + for domain in domains: + validate_existing_invitation(email, domain) + + +def check_outside_org_membership(email, requestor, is_member_of_different_org): + """Raise an error if the email belongs to a different organization.""" if ( flag_is_active_for_user(requestor, "organization_feature") and not flag_is_active_for_user(requestor, "multiple_portfolios") and is_member_of_different_org ): - raise OutsideOrgMemberError + raise OutsideOrgMemberError(email=email) - # Check for an existing invitation for each domain - for domain in domains: - try: - invite = DomainInvitation.objects.get(email=email, domain=domain) - if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: - raise AlreadyDomainManagerError(email) - elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: - invite.update_cancellation_status() - invite.save() - else: - raise AlreadyDomainInvitedError(email) - except DomainInvitation.DoesNotExist: - pass - # Send the email +def validate_existing_invitation(email, domain): + """Check for existing invitations and handle their status.""" + try: + invite = DomainInvitation.objects.get(email=email, domain=domain) + if invite.status == DomainInvitation.DomainInvitationStatus.RETRIEVED: + raise AlreadyDomainManagerError(email) + elif invite.status == DomainInvitation.DomainInvitationStatus.CANCELED: + invite.update_cancellation_status() + invite.save() + else: + raise AlreadyDomainInvitedError(email) + except DomainInvitation.DoesNotExist: + pass + + +def send_invitation_email(email, requestor_email, domains, requested_user): + """Send the invitation email.""" try: send_templated_email( "emails/domain_invitation.txt", @@ -88,9 +108,7 @@ def send_domain_invitation_email(email: str, requestor, domains: Domain | list[D ) except EmailSendingError as err: domain_names = ", ".join([domain.name for domain in domains]) - raise EmailSendingError( - f"Could not send email invitation to {email} for domains: {domain_names}" - ) from err + raise EmailSendingError(f"Could not send email invitation to {email} for domains: {domain_names}") from err def send_portfolio_invitation_email(email: str, requestor, portfolio): diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b16e3681d..2c5553a92 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -1186,6 +1186,7 @@ class DomainAddUserView(DomainFormBaseView): portfolio_invitation, _ = PortfolioInvitation.objects.get_or_create( email=requested_email, portfolio=domain_org, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] ) + # if user exists for email, immediately retrieve portfolio invitation upon creation if requested_user is not None: portfolio_invitation.retrieve() portfolio_invitation.save() diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 60a96bba6..b9249ca6d 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -296,7 +296,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V Processes added domains by bulk creating UserDomainRole instances. """ if added_domain_ids: - print('_process_added_domains') + # get added_domains from ids to pass to send email method and bulk create added_domains = Domain.objects.filter(id__in=added_domain_ids) member_of_a_different_org, _ = get_org_membership(portfolio, member.email, member) send_domain_invitation_email( @@ -514,14 +514,15 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission or creating new ones. """ if added_domain_ids: + # get added_domains from ids to pass to send email method and bulk create added_domains = Domain.objects.filter(id__in=added_domain_ids) member_of_a_different_org, _ = get_org_membership(portfolio, email, None) send_domain_invitation_email( - email=email, - requestor=requestor, - domains=added_domains, - is_member_of_different_org=member_of_a_different_org, - ) + email=email, + requestor=requestor, + domains=added_domains, + is_member_of_different_org=member_of_a_different_org, + ) # Update existing invitations from CANCELED to INVITED existing_invitations = DomainInvitation.objects.filter(domain__in=added_domains, email=email) @@ -542,7 +543,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission for domain_id in new_domain_ids ] ) - + def _process_removed_domains(self, removed_domain_ids, email): """ Processes removed domain invitations by updating their status to CANCELED. @@ -777,6 +778,7 @@ class PortfolioAddMemberView(PortfolioMembersPermissionView, FormMixin): if not requested_user or not permission_exists: send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=portfolio) portfolio_invitation = form.save() + # if user exists for email, immediately retrieve portfolio invitation upon creation if requested_user is not None: portfolio_invitation.retrieve() portfolio_invitation.save() diff --git a/src/registrar/views/utility/invitation_helper.py b/src/registrar/views/utility/invitation_helper.py index 2cdde6a5e..771300406 100644 --- a/src/registrar/views/utility/invitation_helper.py +++ b/src/registrar/views/utility/invitation_helper.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) # when creating invitations and sending associated emails. These can be reused in # any view, and were initially developed for domain.py, portfolios.py and admin.py + def get_org_membership(requestor_org, requested_email, requested_user): """ Verifies if an email belongs to a different organization as a member or invited member. From a302fd8e9ebdeb7da9cd35f445dc74a54a87d905 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 9 Jan 2025 12:39:46 -0500 Subject: [PATCH 12/58] move scss around to account for a netsing rule deprecation --- src/registrar/assets/src/sass/_theme/_tooltips.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/src/sass/_theme/_tooltips.scss b/src/registrar/assets/src/sass/_theme/_tooltips.scss index 58beb8ae6..65bfbb483 100644 --- a/src/registrar/assets/src/sass/_theme/_tooltips.scss +++ b/src/registrar/assets/src/sass/_theme/_tooltips.scss @@ -66,9 +66,9 @@ text-align: center; font-size: inherit; //inherit tooltip fontsize of .93rem max-width: fit-content; + display: block; @include at-media('desktop') { width: 70vw; } - display: block; } } \ No newline at end of file From 1b6667ab73097a63ed869f76e71391291ee92a10 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:01:15 -0700 Subject: [PATCH 13/58] Cleanup script --- .../commands/create_federal_portfolio.py | 10 +- .../commands/patch_suborganizations.py | 216 +++++++----------- .../commands/utility/terminal_helper.py | 19 +- .../models/utility/generic_helper.py | 15 ++ 4 files changed, 107 insertions(+), 153 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 37bd4765d..43030d078 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -133,15 +133,15 @@ class Command(BaseCommand): portfolio__isnull=True, organization_name__isnull=True, sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name") - ).in_bulk("organization_name") + organization_name__iexact=F("portfolio__organization_name"), + ).in_bulk(field_name="organization_name") requests = requests.exclude( portfolio__isnull=True, organization_name__isnull=True, sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name") - ).in_bulk("organization_name") + organization_name__iexact=F("portfolio__organization_name"), + ).in_bulk(field_name="organization_name") for suborg in suborganizations: domain = domains.get(suborg.name, None) @@ -169,7 +169,7 @@ class Command(BaseCommand): if city: suborg.city = city - + if suborg: suborg.state_territory = state_territory diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 07fb94f3b..2c471aad9 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -1,9 +1,8 @@ -@ -0,0 +1,123 @@ import logging from django.core.management import BaseCommand from registrar.models import Suborganization, DomainRequest, DomainInformation from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper -from registrar.models.utility.generic_helper import normalize_string +from registrar.models.utility.generic_helper import count_capitals, normalize_string logger = logging.getLogger(__name__) @@ -13,157 +12,98 @@ class Command(BaseCommand): help = "Clean up duplicate suborganizations that differ only by spaces and capitalization" def handle(self, **kwargs): - manual_records = [ - "Assistant Secretary for Preparedness and Response Office of the Secretary", - "US Geological Survey", - "USDA/OC", - ] - duplicates = {} - for record in Suborganization.objects.filter(name__in=manual_records): - if record.name: - norm_name = normalize_string(record.name) - duplicates[norm_name] = { - "keep": None, - "delete": [record] - } + """Process manual deletions and find/remove duplicates. Shows preview + and updates DomainInformation / DomainRequest sub_organization references before deletion.""" - records_to_delete.update(self.handle_suborganization_duplicates()) + # First: get a preset list of records we want to delete. + # The key gets deleted, the value gets kept. + additional_records_to_delete = { + normalize_string("Assistant Secretary for Preparedness and Response Office of the Secretary"): { + "keep": Suborganization.objects.none() + }, + normalize_string("US Geological Survey"): {"keep": Suborganization.objects.none()}, + normalize_string("USDA/OC"): {"keep": Suborganization.objects.none()}, + } - # Get confirmation and execute deletions - if TerminalHelper.prompt_for_execution( - system_exit_on_terminate=True, - prompt_message=preview, - prompt_title="Clean up duplicate suborganizations?", - verify_message="*** WARNING: This will delete suborganizations! ***" - ): - # Update all references to point to the right suborg before deletion - for record in duplicates.values(): - best_record = record.get("keep") - delete_ids = [dupe.id for dupe in record.get("delete")] - - # Update domain requests - DomainRequest.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - # Update domain information - DomainInformation.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) + # First: Group all suborganization names by their "normalized" names (finding duplicates) + name_groups = {} + for suborg in Suborganization.objects.all(): + normalized_name = normalize_string(suborg.name) + if normalized_name not in name_groups: + name_groups[normalized_name] = [] + name_groups[normalized_name].append(suborg) - records_to_delete = set( - dupe.id - for data in duplicates.values() - for dupe in data["delete"] - ) - try: - delete_count, _ = Suborganization.objects.filter(id__in=records_to_delete).delete() - logger.info(f"{TerminalColors.OKGREEN}Successfully deleted {delete_count} suborganizations{TerminalColors.ENDC}") - except Exception as e: - logger.error(f"{TerminalColors.FAIL}Failed to clean up suborganizations: {str(e)}{TerminalColors.ENDC}") - - - def handle_suborganization_duplicates(self, duplicates): - # Find duplicates - all_suborgs = Suborganization.objects.all() - for suborg in all_suborgs: - # Normalize name by removing extra spaces and converting to lowercase - normalized_name = " ".join(suborg.name.split()).lower() - - # First occurrence of this name - if normalized_name not in duplicates: - duplicates[normalized_name] = { - "keep": suborg, - "delete": [] - } + # Second: find the record we should keep, and the duplicate records we should delete + records_to_prune = {} + for normalized_name, duplicate_suborgs in name_groups.items(): + if normalized_name in additional_records_to_delete: + record = additional_records_to_delete.get(normalized_name) + records_to_prune[normalized_name] = {"keep": record.get("keep"), "delete": duplicate_suborgs} continue - # Compare with our current best - current_best = duplicates[normalized_name]["keep"] + if len(duplicate_suborgs) > 1: + # Pick the best record to keep. + # The fewest spaces and most capitals (at the beginning of each word) wins. + best_record = duplicate_suborgs[0] + for suborg in duplicate_suborgs: + has_fewer_spaces = suborg.name.count(" ") < best_record.name.count(" ") + has_more_capitals = count_capitals(suborg.name, leading_only=True) > count_capitals( + best_record.name, leading_only=True + ) + if has_fewer_spaces or has_more_capitals: + best_record = suborg - # Check if all other fields match. - # If they don't, we should inspect this record manually. - fields_to_compare = ["portfolio", "city", "state_territory"] - fields_match = all( - getattr(suborg, field) == getattr(current_best, field) - for field in fields_to_compare - ) - if not fields_match: - logger.warning( - f"{TerminalColors.YELLOW}" - f"\nSkipping potential duplicate: {suborg.name} (id: {suborg.id})" - f"\nData mismatch with {current_best.name} (id: {current_best.id})" - f"{TerminalColors.ENDC}" - ) - continue - - # Determine if new suborg is better than current best. - # The fewest spaces and most capitals wins. - new_has_fewer_spaces = suborg.name.count(" ") < current_best.name.count(" ") - new_has_more_capitals = sum(1 for c in suborg.name if c.isupper()) > sum(1 for c in current_best.name if c.isupper()) - # TODO - # Split into words and count properly capitalized first letters - # new_proper_caps = sum( - # 1 for word in suborg.name.split() - # if word and word[0].isupper() - # ) - # current_proper_caps = sum( - # 1 for word in current_best.name.split() - # if word and word[0].isupper() - # ) - # new_has_better_caps = new_proper_caps > current_proper_caps + records_to_prune[normalized_name] = { + "keep": best_record, + "delete": [s for s in duplicate_suborgs if s != best_record], + } - if new_has_fewer_spaces or new_has_more_capitals: - # New suborg is better - demote the old one to the delete list - duplicates[normalized_name]["delete"].append(current_best) - duplicates[normalized_name]["keep"] = suborg - else: - # If it is not better, just delete the old one - duplicates[normalized_name]["delete"].append(suborg) - - # Filter out entries without duplicates - duplicates = {k: v for k, v in duplicates.items() if v.get("delete")} - if not duplicates: - logger.info(f"No duplicate suborganizations found.") + if len(records_to_prune) == 0: + TerminalHelper.colorful_logger(logger.error, TerminalColors.FAIL, "No suborganizations to delete.") return - # Show preview of changes - preview = "The following duplicates will be removed:\n" - for data in duplicates.values(): - best = data.get("keep") - preview += f"\nKeeping: '{best.name}' (id: {best.id})" - + # Third: Show a preview of the changes + total_records_to_remove = 0 + preview = "The following records will be removed:\n" + for data in records_to_prune.values(): + keep = data.get("keep") + if keep: + preview += f"\nKeeping: '{keep.name}' (id: {keep.id})" + for duplicate in data.get("delete"): preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" + total_records_to_remove += 1 preview += "\n" - # Get confirmation and execute deletions + # Fourth: Get user confirmation and execute deletions if TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, prompt_message=preview, - prompt_title="Clean up duplicate suborganizations?", - verify_message="*** WARNING: This will delete suborganizations! ***" + prompt_title=f"Remove {total_records_to_remove} suborganizations?", + verify_message="*** WARNING: This will replace the record on DomainInformation and DomainRequest! ***", ): - # Update all references to point to the right suborg before deletion - for record in duplicates.values(): - best_record = record.get("keep") - delete_ids = [dupe.id for dupe in record.get("delete")] - - # Update domain requests - DomainRequest.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) - - # Update domain information - DomainInformation.objects.filter( - sub_organization_id__in=delete_ids - ).update(sub_organization=best_record) + try: + # Update all references to point to the right suborg before deletion + all_suborgs_to_remove = set() + for record in records_to_prune.values(): + best_record = record["keep"] + suborgs_to_remove = {dupe.id for dupe in record["delete"]} + # Update domain requests + DomainRequest.objects.filter(sub_organization_id__in=suborgs_to_remove).update( + sub_organization=best_record + ) - records_to_delete = set( - dupe.id - for data in duplicates.values() - for dupe in data["delete"] - ) - return records_to_delete - else: - return set() + # Update domain information + DomainInformation.objects.filter(sub_organization_id__in=suborgs_to_remove).update( + sub_organization=best_record + ) + + all_suborgs_to_remove.update(suborgs_to_remove) + delete_count, _ = Suborganization.objects.filter(id__in=all_suborgs_to_remove).delete() + TerminalHelper.colorful_logger( + logger.info, TerminalColors.MAGENTA, f"Successfully deleted {delete_count} suborganizations." + ) + except Exception as e: + TerminalHelper.colorful_logger( + logger.error, TerminalColors.FAIL, f"Failed to delete suborganizations: {str(e)}" + ) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index eed1027f7..b16ca72f2 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -401,16 +401,15 @@ class TerminalHelper: # Allow the user to inspect the command string # and ask if they wish to proceed proceed_execution = TerminalHelper.query_yes_no_exit( - f"""{TerminalColors.OKCYAN} - ===================================================== - {prompt_title} - ===================================================== - {verify_message} - - {prompt_message} - {TerminalColors.FAIL} - Proceed? (Y = proceed, N = {action_description_for_selecting_no}) - {TerminalColors.ENDC}""" + f"\n{TerminalColors.OKCYAN}" + "=====================================================" + f"\n{prompt_title}\n" + "=====================================================" + f"\n{verify_message}\n" + f"\n{prompt_message}\n" + f"{TerminalColors.FAIL}" + f"Proceed? (Y = proceed, N = {action_description_for_selecting_no})" + f"{TerminalColors.ENDC}" ) # If the user decided to proceed return true. diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 84dc28db1..af7780194 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -344,6 +344,7 @@ def value_of_attribute(obj, attribute_name: str): value = value() return value + def normalize_string(string_to_normalize, lowercase=True): """Normalizes a given string. Returns a string without extra spaces, in all lowercase.""" if not isinstance(string_to_normalize, str): @@ -352,3 +353,17 @@ def normalize_string(string_to_normalize, lowercase=True): new_string = " ".join(string_to_normalize.split()) return new_string.lower() if lowercase else new_string + + +def count_capitals(text: str, leading_only: bool): + """Counts capital letters in a string. + Args: + text (str): The string to analyze. + leading_only (bool): If False, counts all capital letters. + If True, only counts capitals at the start of words. + Returns: + int: Number of capital letters found. + """ + if leading_only: + return sum(word[0].isupper() for word in text.split() if word) + return sum(c.isupper() for c in text if c) From c1eea3cfd99256e4c0305d5f93cf4f60ede66284 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 13:47:57 -0500 Subject: [PATCH 14/58] updated some broken tests --- src/registrar/tests/test_admin.py | 15 +++++++++++-- src/registrar/tests/test_views_portfolio.py | 25 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 3195f8237..8aeb36961 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -136,12 +136,17 @@ class TestDomainInvitationAdmin(TestCase): self.admin = ListHeaderAdmin(model=DomainInvitationAdmin, admin_site=AdminSite()) self.superuser = create_superuser() self.domain = Domain.objects.create(name="example.com") + self.portfolio = Portfolio.objects.create(organization_name="new portfolio", creator=self.superuser) + DomainInformation.objects.create(domain=self.domain, portfolio=self.portfolio, creator=self.superuser) """Create a client object""" self.client = Client(HTTP_HOST="localhost:8080") def tearDown(self): """Delete all DomainInvitation objects""" DomainInvitation.objects.all().delete() + DomainInformation.objects.all().delete() + Portfolio.objects.all().delete() + Domain.objects.all().delete() Contact.objects.all().delete() User.objects.all().delete() @@ -189,7 +194,10 @@ class TestDomainInvitationAdmin(TestCase): self.assertContains(response, retrieved_html, count=1) @less_console_noise_decorator - def test_save_model_user_exists(self): + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_save_model_user_exists(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): """Test saving a domain invitation when the user exists. Should attempt to retrieve the domain invitation.""" @@ -217,7 +225,10 @@ class TestDomainInvitationAdmin(TestCase): self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") @less_console_noise_decorator - def test_save_model_user_does_not_exist(self): + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_save_model_user_does_not_exist(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): """Test saving a domain invitation when the user does not exist. Should not attempt to retrieve the domain invitation.""" diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 9bc97874d..b6b275c99 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2180,7 +2180,8 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) - def test_post_with_valid_added_domains(self): + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_with_valid_added_domains(self, mock_send_domain_email): """Test that domains can be successfully added.""" self.client.force_login(self.user) @@ -2198,6 +2199,15 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "The domain assignment changes have been saved.") + expected_domains = Domain.objects.filter(id__in=[1, 2, 3]) + # Verify that the invitation email was sent + mock_send_domain_email.assert_called_once() + call_args = mock_send_domain_email.call_args.kwargs + self.assertEqual(call_args["email"], "info@example.com") + self.assertEqual(call_args["requestor"], self.user) + self.assertEqual(list(call_args["domains"]), list(expected_domains)) + self.assertIsNone(call_args.get("is_member_of_different_org")) + @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) @@ -2364,7 +2374,8 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) - def test_post_with_valid_added_domains(self): + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_with_valid_added_domains(self, mock_send_domain_email): """Test adding new domains successfully.""" self.client.force_login(self.user) @@ -2387,6 +2398,16 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "The domain assignment changes have been saved.") + expected_domains = Domain.objects.filter(id__in=[1, 2, 3]) + # Verify that the invitation email was sent + mock_send_domain_email.assert_called_once() + call_args = mock_send_domain_email.call_args.kwargs + self.assertEqual(call_args["email"], "info@example.com") + self.assertEqual(call_args["requestor"], self.user) + self.assertEqual(list(call_args["domains"]), list(expected_domains)) + self.assertIsNone(call_args.get("is_member_of_different_org")) + + @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) From f170ab90ca8d6e833b53237316df9120f4bb79bc Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:17:49 -0700 Subject: [PATCH 15/58] Add records to delete manually --- .../commands/patch_suborganizations.py | 85 +++++++++++-------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 2c471aad9..8769ed358 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -16,16 +16,25 @@ class Command(BaseCommand): and updates DomainInformation / DomainRequest sub_organization references before deletion.""" # First: get a preset list of records we want to delete. - # The key gets deleted, the value gets kept. - additional_records_to_delete = { + # For extra_records_to_prune: the key gets deleted, the value gets kept. + extra_records_to_prune = { normalize_string("Assistant Secretary for Preparedness and Response Office of the Secretary"): { - "keep": Suborganization.objects.none() + "keep": "Assistant Secretary for Preparedness and Response, Office of the Secretary" }, - normalize_string("US Geological Survey"): {"keep": Suborganization.objects.none()}, - normalize_string("USDA/OC"): {"keep": Suborganization.objects.none()}, + normalize_string("US Geological Survey"): {"keep": "U.S. Geological Survey"}, + normalize_string("USDA/OC"): {"keep": "USDA, Office of Communications"}, + normalize_string("GSA, IC, OGP WebPortfolio"): {"keep": "GSA, IC, OGP Web Portfolio"}, + normalize_string("USDA/ARS/NAL"): {"keep": "USDA, ARS, NAL"} + # TODO - U.S Immigration and Customs Enforcement } - # First: Group all suborganization names by their "normalized" names (finding duplicates) + # First: Group all suborganization names by their "normalized" names (finding duplicates). + # Returns a dict that looks like this: + # { + # "amtrak": [, , ], + # ...etc + # } + # name_groups = {} for suborg in Suborganization.objects.all(): normalized_name = normalize_string(suborg.name) @@ -33,26 +42,35 @@ class Command(BaseCommand): name_groups[normalized_name] = [] name_groups[normalized_name].append(suborg) - # Second: find the record we should keep, and the duplicate records we should delete + # Second: find the record we should keep, and the records we should delete + # Returns a dict that looks like this: + # { + # "amtrak": { + # "keep": + # "delete": [, ] + # }, + # "usda/oc": { + # "keep": , + # "delete": [] + # }, + # ...etc + # } records_to_prune = {} for normalized_name, duplicate_suborgs in name_groups.items(): - if normalized_name in additional_records_to_delete: - record = additional_records_to_delete.get(normalized_name) - records_to_prune[normalized_name] = {"keep": record.get("keep"), "delete": duplicate_suborgs} - continue - - if len(duplicate_suborgs) > 1: - # Pick the best record to keep. - # The fewest spaces and most capitals (at the beginning of each word) wins. - best_record = duplicate_suborgs[0] - for suborg in duplicate_suborgs: - has_fewer_spaces = suborg.name.count(" ") < best_record.name.count(" ") - has_more_capitals = count_capitals(suborg.name, leading_only=True) > count_capitals( - best_record.name, leading_only=True - ) - if has_fewer_spaces or has_more_capitals: - best_record = suborg + # Delete data from our preset list + if normalized_name in extra_records_to_prune: + record = extra_records_to_prune.get(normalized_name) + # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. + hardcoded_record_name = normalize_string(record.get("keep")) + name_group = name_groups.get(hardcoded_record_name, []) + # This assumes that there is only one item in the name_group array. Should be fine, given our data. + keep = name_group[0] if name_group else None + records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} + # Delete duplicates (extra spaces or casing differences) + elif len(duplicate_suborgs) > 1: + # Pick the best record (fewest spaces, most leading capitals) + best_record = max(duplicate_suborgs, key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True))) records_to_prune[normalized_name] = { "keep": best_record, "delete": [s for s in duplicate_suborgs if s != best_record], @@ -62,18 +80,20 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.error, TerminalColors.FAIL, "No suborganizations to delete.") return - # Third: Show a preview of the changes + # Third: Build a preview of the changes total_records_to_remove = 0 - preview = "The following records will be removed:\n" + preview_lines = ["The following records will be removed:"] for data in records_to_prune.values(): keep = data.get("keep") + delete = data.get("delete") if keep: - preview += f"\nKeeping: '{keep.name}' (id: {keep.id})" - - for duplicate in data.get("delete"): - preview += f"\nRemoving: '{duplicate.name}' (id: {duplicate.id})" + preview_lines.append(f"Keeping: '{keep.name}' (id: {keep.id})") + + for duplicate in delete: + preview_lines.append(f"Removing: '{duplicate.name}' (id: {duplicate.id})") total_records_to_remove += 1 - preview += "\n" + preview_lines.append("") + preview = "\n".join(preview_lines) # Fourth: Get user confirmation and execute deletions if TerminalHelper.prompt_for_execution( @@ -88,17 +108,14 @@ class Command(BaseCommand): for record in records_to_prune.values(): best_record = record["keep"] suborgs_to_remove = {dupe.id for dupe in record["delete"]} - # Update domain requests DomainRequest.objects.filter(sub_organization_id__in=suborgs_to_remove).update( sub_organization=best_record ) - - # Update domain information DomainInformation.objects.filter(sub_organization_id__in=suborgs_to_remove).update( sub_organization=best_record ) - all_suborgs_to_remove.update(suborgs_to_remove) + # Delete the suborgs delete_count, _ = Suborganization.objects.filter(id__in=all_suborgs_to_remove).delete() TerminalHelper.colorful_logger( logger.info, TerminalColors.MAGENTA, f"Successfully deleted {delete_count} suborganizations." From 028d033ff2bcdebe5c403a79e5dd17d4913a9f09 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:24:03 -0700 Subject: [PATCH 16/58] Code comments + lint --- .../commands/patch_suborganizations.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 8769ed358..82d5bd689 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -24,16 +24,16 @@ class Command(BaseCommand): normalize_string("US Geological Survey"): {"keep": "U.S. Geological Survey"}, normalize_string("USDA/OC"): {"keep": "USDA, Office of Communications"}, normalize_string("GSA, IC, OGP WebPortfolio"): {"keep": "GSA, IC, OGP Web Portfolio"}, - normalize_string("USDA/ARS/NAL"): {"keep": "USDA, ARS, NAL"} - # TODO - U.S Immigration and Customs Enforcement + normalize_string("USDA/ARS/NAL"): {"keep": "USDA, ARS, NAL"}, } # First: Group all suborganization names by their "normalized" names (finding duplicates). # Returns a dict that looks like this: # { # "amtrak": [, , ], + # "usda/oc": [], # ...etc - # } + # } # name_groups = {} for suborg in Suborganization.objects.all(): @@ -59,18 +59,19 @@ class Command(BaseCommand): for normalized_name, duplicate_suborgs in name_groups.items(): # Delete data from our preset list if normalized_name in extra_records_to_prune: - record = extra_records_to_prune.get(normalized_name) # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. - hardcoded_record_name = normalize_string(record.get("keep")) - name_group = name_groups.get(hardcoded_record_name, []) - - # This assumes that there is only one item in the name_group array. Should be fine, given our data. + # This assumes that there is only one item in the name_group array (see usda/oc example). Should be fine, given our data. + hardcoded_record_name = extra_records_to_prune[normalized_name]["keep"] + name_group = name_groups.get(normalized_name(hardcoded_record_name), []) keep = name_group[0] if name_group else None records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} # Delete duplicates (extra spaces or casing differences) elif len(duplicate_suborgs) > 1: # Pick the best record (fewest spaces, most leading capitals) - best_record = max(duplicate_suborgs, key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True))) + best_record = max( + duplicate_suborgs, + key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True)), + ) records_to_prune[normalized_name] = { "keep": best_record, "delete": [s for s in duplicate_suborgs if s != best_record], @@ -88,14 +89,14 @@ class Command(BaseCommand): delete = data.get("delete") if keep: preview_lines.append(f"Keeping: '{keep.name}' (id: {keep.id})") - + for duplicate in delete: preview_lines.append(f"Removing: '{duplicate.name}' (id: {duplicate.id})") total_records_to_remove += 1 preview_lines.append("") preview = "\n".join(preview_lines) - # Fourth: Get user confirmation and execute deletions + # Fourth: Get user confirmation and delete if TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, prompt_message=preview, From 632eccf7e9a6fc40fbf0b1e6de8abb3ba7f48537 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 15:45:52 -0500 Subject: [PATCH 17/58] a few updates to tests --- src/registrar/tests/test_reports.py | 1 + src/registrar/tests/test_views_domain.py | 2 +- src/registrar/tests/test_views_portfolio.py | 7 ++++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 4a41238c7..b11500ea9 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -900,6 +900,7 @@ class MemberExportTest(MockDbForIndividualTests, MockEppLib): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.maxDiff = None self.assertEqual(csv_content, expected_content) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index aedfc41c2..a3e324d42 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -759,7 +759,7 @@ class TestDomainManagers(TestDomainOverview): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() - self.assertContains(success_page, "Could not send email invitation.") + self.assertContains(success_page, "Failed to send email.") @boto3_mocking.patching @less_console_noise_decorator diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index b6b275c99..18f68a7fb 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2402,16 +2402,17 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain # Verify that the invitation email was sent mock_send_domain_email.assert_called_once() call_args = mock_send_domain_email.call_args.kwargs - self.assertEqual(call_args["email"], "info@example.com") + self.assertEqual(call_args["email"], "invited@example.com") self.assertEqual(call_args["requestor"], self.user) self.assertEqual(list(call_args["domains"]), list(expected_domains)) - self.assertIsNone(call_args.get("is_member_of_different_org")) + self.assertFalse(call_args.get("is_member_of_different_org")) @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) - def test_post_with_existing_and_new_added_domains(self): + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_with_existing_and_new_added_domains(self, mock_send_domain_email): """Test updating existing and adding new invitations.""" self.client.force_login(self.user) From db75d31a0b93d5f267c28888d639dd0f60a2690b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:22:47 -0700 Subject: [PATCH 18/58] Wrap up PR --- docs/operations/data_migration.md | 35 ++++ .../commands/create_federal_portfolio.py | 77 ++++++--- .../commands/patch_suborganizations.py | 14 +- .../models/utility/generic_helper.py | 2 +- src/registrar/tests/common.py | 2 + .../tests/test_management_scripts.py | 159 +++++++++++++++++- 6 files changed, 252 insertions(+), 37 deletions(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 0863aa0b7..499c0840f 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -918,3 +918,38 @@ Example (only requests): `./manage.py create_federal_portfolio --branch "executi - Parameters #1-#2: Either `--agency_name` or `--branch` must be specified. Not both. - Parameters #2-#3, you cannot use `--both` while using these. You must specify either `--parse_requests` or `--parse_domains` seperately. While all of these parameters are optional in that you do not need to specify all of them, you must specify at least one to run this script. + + +## Patch suborganizations +This script deletes some duplicate suborganization data that exists in our database (one-time use). +It works in two ways: +1. If the only name difference between two suborg records is extra spaces or a capitalization difference, +then we delete all duplicate records of this type. +2. If the suborg name is one we manually specify to delete via the script. + +Before it deletes records, it goes through each DomainInformation and DomainRequest object and updates the reference to "sub_organization" to match the non-duplicative record. + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-za` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Upload your csv to the desired sandbox +[Follow these steps](#use-scp-to-transfer-data-to-sandboxes) to upload the federal_cio csv to a sandbox of your choice. + +#### Step 5: Running the script +To create a specific portfolio: +```./manage.py patch_suborganizations``` + +### Running locally + +#### Step 1: Running the script +```docker-compose exec app ./manage.py patch_suborganizations``` diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 43030d078..31f4b1e28 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -80,18 +80,33 @@ class Command(BaseCommand): else: raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") + all_suborganizations = [] + all_domains = [] + all_domain_requests = [] for federal_agency in agencies: message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) try: # C901 'Command.handle' is too complex (12) - self.handle_populate_portfolio(federal_agency, parse_domains, parse_requests, both) + suborgs, domains, requests = self.handle_populate_portfolio( + federal_agency, parse_domains, parse_requests, both + ) + all_suborganizations.extend(suborgs) + all_domains.extend(domains) + all_domain_requests.extend(requests) except Exception as exec: self.failed_portfolios.add(federal_agency) logger.error(exec) message = f"Failed to create portfolio '{federal_agency.agency}'" TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) + # Post process steps + # Add suborg info to created or existing suborgs. + if all_suborganizations: + self.post_process_suborganization_fields(all_suborganizations, all_domains, all_domain_requests) + message = f"Added city and state_territory information to {len(all_suborganizations)} suborgs." + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + TerminalHelper.log_script_run_summary( self.updated_portfolios, self.failed_portfolios, @@ -105,47 +120,46 @@ class Command(BaseCommand): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" portfolio, created = self.create_portfolio(federal_agency) - suborganizations = Suborganization.objects.none() - domains = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) - domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( - status__in=[ - DomainRequest.DomainRequestStatus.STARTED, - DomainRequest.DomainRequestStatus.INELIGIBLE, - DomainRequest.DomainRequestStatus.REJECTED, - ] - ) + suborganizations = self.create_suborganizations(portfolio, federal_agency) + domains = [] + domain_requests = [] if created: - suborganizations = self.create_suborganizations(portfolio, federal_agency) if parse_domains or both: - domains = self.handle_portfolio_domains(portfolio, federal_agency, domains) + domains = self.handle_portfolio_domains(portfolio, federal_agency) if parse_requests or both: - domain_requests = self.handle_portfolio_requests(portfolio, federal_agency, domain_requests) + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) - # Post process steps - # Add suborg info to created or existing suborgs. Get the refreshed queryset for each. - self.post_process_suborganization_fields(suborganizations.all(), domains.all(), domain_requests.all()) + return suborganizations, domains, domain_requests def post_process_suborganization_fields(self, suborganizations, domains, requests): + """Post-process suborganization fields by pulling data from related domains and requests. + + This function updates suborganization city and state_territory fields based on + related domain information and domain request information. + """ + # Exclude domains and requests where the org name is the same, # and where we are missing some crucial information. - domains = domains.exclude( + domains = DomainInformation.objects.filter(id__in=[domain.id for domain in domains]).exclude( portfolio__isnull=True, organization_name__isnull=True, sub_organization__isnull=True, organization_name__iexact=F("portfolio__organization_name"), - ).in_bulk(field_name="organization_name") + ) + domains_dict = {domain.organization_name: domain for domain in domains} - requests = requests.exclude( + requests = DomainRequest.objects.filter(id__in=[request.id for request in requests]).exclude( portfolio__isnull=True, organization_name__isnull=True, sub_organization__isnull=True, organization_name__iexact=F("portfolio__organization_name"), - ).in_bulk(field_name="organization_name") + ) + requests_dict = {request.organization_name: request for request in requests} for suborg in suborganizations: - domain = domains.get(suborg.name, None) - request = requests.get(suborg.name, None) + domain = domains_dict.get(suborg.name, None) + request = requests_dict.get(suborg.name, None) # PRIORITY: # 1. Domain info @@ -266,11 +280,19 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") return new_suborgs - def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_requests): + def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): """ Associate portfolio with domain requests for a federal agency. Updates all relevant domain request records. """ + domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( + status__in=[ + DomainRequest.DomainRequestStatus.STARTED, + DomainRequest.DomainRequestStatus.INELIGIBLE, + DomainRequest.DomainRequestStatus.REJECTED, + ] + ) + if not domain_requests.exists(): message = f""" Portfolio '{portfolio}' not added to domain requests: no valid records found. @@ -295,14 +317,14 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - # Return a fresh copy of the queryset - return domain_requests.all() + return domain_requests - def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency, domain_infos): + def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): """ Associate portfolio with domains for a federal agency. Updates all relevant domain information records. """ + domain_infos = DomainInformation.objects.filter(federal_agency=federal_agency, portfolio__isnull=True) if not domain_infos.exists(): message = f""" Portfolio '{portfolio}' not added to domains: no valid records found. @@ -323,5 +345,4 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - # Return a fresh copy of the queryset - return domain_infos.all() + return domain_infos diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 82d5bd689..b06cbb05b 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -19,12 +19,12 @@ class Command(BaseCommand): # For extra_records_to_prune: the key gets deleted, the value gets kept. extra_records_to_prune = { normalize_string("Assistant Secretary for Preparedness and Response Office of the Secretary"): { - "keep": "Assistant Secretary for Preparedness and Response, Office of the Secretary" + "replace_with": "Assistant Secretary for Preparedness and Response, Office of the Secretary" }, - normalize_string("US Geological Survey"): {"keep": "U.S. Geological Survey"}, - normalize_string("USDA/OC"): {"keep": "USDA, Office of Communications"}, - normalize_string("GSA, IC, OGP WebPortfolio"): {"keep": "GSA, IC, OGP Web Portfolio"}, - normalize_string("USDA/ARS/NAL"): {"keep": "USDA, ARS, NAL"}, + normalize_string("US Geological Survey"): {"replace_with": "U.S. Geological Survey"}, + normalize_string("USDA/OC"): {"replace_with": "USDA, Office of Communications"}, + normalize_string("GSA, IC, OGP WebPortfolio"): {"replace_with": "GSA, IC, OGP Web Portfolio"}, + normalize_string("USDA/ARS/NAL"): {"replace_with": "USDA, ARS, NAL"}, } # First: Group all suborganization names by their "normalized" names (finding duplicates). @@ -61,8 +61,8 @@ class Command(BaseCommand): if normalized_name in extra_records_to_prune: # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. # This assumes that there is only one item in the name_group array (see usda/oc example). Should be fine, given our data. - hardcoded_record_name = extra_records_to_prune[normalized_name]["keep"] - name_group = name_groups.get(normalized_name(hardcoded_record_name), []) + hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] + name_group = name_groups.get(normalize_string(hardcoded_record_name)) keep = name_group[0] if name_group else None records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} # Delete duplicates (extra spaces or casing differences) diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index af7780194..c2ba6887f 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -355,7 +355,7 @@ def normalize_string(string_to_normalize, lowercase=True): return new_string.lower() if lowercase else new_string -def count_capitals(text: str, leading_only: bool): +def count_capitals(text: str, leading_only: bool): """Counts capital letters in a string. Args: text (str): The string to analyze. diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 8eca0108e..48806cc7e 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -39,6 +39,7 @@ from epplibwrapper import ( ErrorCode, responses, ) +from registrar.models.suborganization import Suborganization from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices from registrar.models.user_domain_role import UserDomainRole @@ -905,6 +906,7 @@ class MockDb(TestCase): DomainInformation.objects.all().delete() DomainRequest.objects.all().delete() UserDomainRole.objects.all().delete() + Suborganization.objects.all().delete() Portfolio.objects.all().delete() UserPortfolioPermission.objects.all().delete() User.objects.all().delete() diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 7cce0d2b2..1be62ab86 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1,4 +1,5 @@ import copy +from io import StringIO import boto3_mocking # type: ignore from datetime import date, datetime, time from django.core.management import call_command @@ -32,7 +33,7 @@ import tablib from unittest.mock import patch, call, MagicMock, mock_open from epplibwrapper import commands, common -from .common import MockEppLib, less_console_noise, completed_domain_request, MockSESClient +from .common import MockEppLib, less_console_noise, completed_domain_request, MockSESClient, MockDbForIndividualTests from api.tests.common import less_console_noise_decorator @@ -1731,3 +1732,159 @@ class TestCreateFederalPortfolio(TestCase): self.assertEqual(existing_portfolio.organization_name, self.federal_agency.agency) self.assertEqual(existing_portfolio.notes, "Old notes") self.assertEqual(existing_portfolio.creator, self.user) + + def test_post_process_suborganization_fields(self): + """Test suborganization field updates from domain and request data. + Also tests the priority order for updating city and state_territory: + 1. Domain information fields + 2. Domain request suborganization fields + 3. Domain request standard fields + """ + # Create test data with different field combinations + self.domain_info.organization_name = "super" + self.domain_info.city = "Domain City " + self.domain_info.state_territory = "NY" + self.domain_info.save() + + self.domain_request_2.organization_name = "super" + self.domain_request.suborganization_city = "Request Suborg City" + self.domain_request.suborganization_state_territory = "CA" + self.domain_request.city = "Request City" + self.domain_request.state_territory = "TX" + self.domain_request.save() + + # Create another request/info pair without domain info data + self.domain_info_2.organization_name = "creative" + self.domain_info_2.city = None + self.domain_info_2.state_territory = None + self.domain_info_2.save() + + self.domain_request_2.organization_name = "creative" + self.domain_request_2.suborganization_city = "Second Suborg City" + self.domain_request_2.suborganization_state_territory = "WA" + self.domain_request_2.city = "Second City" + self.domain_request_2.state_territory = "OR" + self.domain_request_2.save() + + # Create a third request/info pair without suborg data + self.domain_info_3.organization_name = "names" + self.domain_info_3.city = None + self.domain_info_3.state_territory = None + self.domain_info_3.save() + + self.domain_request_3.organization_name = "names" + self.domain_request_3.suborganization_city = None + self.domain_request_3.suborganization_state_territory = None + self.domain_request_3.city = "Third City" + self.domain_request_3.state_territory = "FL" + self.domain_request_3.save() + + # Test running the script with both, and just with parse_requests + self.run_create_federal_portfolio(agency_name="Test Federal Agency", parse_requests=True, parse_domains=True) + self.run_create_federal_portfolio( + agency_name="Executive Agency 1", + parse_requests=True, + ) + + self.domain_info.refresh_from_db() + self.domain_request.refresh_from_db() + self.domain_info_2.refresh_from_db() + self.domain_request_2.refresh_from_db() + self.domain_info_3.refresh_from_db() + self.domain_request_3.refresh_from_db() + + # Verify suborganizations were created with correct field values + # Should use domain info values + suborg_1 = Suborganization.objects.get(name=self.domain_info.organization_name) + self.assertEqual(suborg_1.city, "Domain City") + self.assertEqual(suborg_1.state_territory, "NY") + + # Should use domain request suborg values + suborg_2 = Suborganization.objects.get(name=self.domain_info_2.organization_name) + self.assertEqual(suborg_2.city, "Second Suborg City") + self.assertEqual(suborg_2.state_territory, "WA") + + # Should use domain request standard values + suborg_3 = Suborganization.objects.get(name=self.domain_info_3.organization_name) + self.assertEqual(suborg_3.city, "Third City") + self.assertEqual(suborg_3.state_territory, "FL") + + +class TestPatchSuborganizations(MockDbForIndividualTests): + """Tests for the patch_suborganizations management command.""" + + @less_console_noise_decorator + def run_patch_suborganizations(self): + """Helper method to run the patch_suborganizations command.""" + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.prompt_for_execution", + return_value=True, + ): + call_command("patch_suborganizations") + + @less_console_noise_decorator + def test_space_and_case_duplicates(self): + """Test cleaning up duplicates that differ by spaces and case. + + Should keep the version with: + 1. Fewest spaces + 2. Most leading capitals + """ + Suborganization.objects.create(name="Test Organization ", portfolio=self.portfolio_1) + Suborganization.objects.create(name="test organization", portfolio=self.portfolio_1) + Suborganization.objects.create(name="Test Organization", portfolio=self.portfolio_1) + + # Create an unrelated record to test that it doesn't get deleted, too + Suborganization.objects.create(name="unrelated org", portfolio=self.portfolio_1) + self.run_patch_suborganizations() + self.assertEqual(Suborganization.objects.count(), 2) + self.assertEqual(Suborganization.objects.filter(name__in=["unrelated org", "Test Organization"]).count(), 2) + + @less_console_noise_decorator + def test_hardcoded_record(self): + """Tests that our hardcoded records update as we expect them to""" + # Create orgs with old and new name formats + old_name = "USDA/OC" + new_name = "USDA, Office of Communications" + + Suborganization.objects.create(name=old_name, portfolio=self.portfolio_1) + Suborganization.objects.create(name=new_name, portfolio=self.portfolio_1) + + self.run_patch_suborganizations() + + # Verify only the new one remains + self.assertEqual(Suborganization.objects.count(), 1) + remaining = Suborganization.objects.first() + self.assertEqual(remaining.name, new_name) + + @less_console_noise_decorator + def test_reference_updates(self): + """Test that references are updated on domain info and domain request before deletion.""" + # Create suborganizations + keep_org = Suborganization.objects.create(name="Test Organization", portfolio=self.portfolio_1) + delete_org = Suborganization.objects.create(name="test organization ", portfolio=self.portfolio_1) + unrelated_org = Suborganization.objects.create(name="awesome", portfolio=self.portfolio_1) + + # We expect these references to update + self.domain_request_1.sub_organization = delete_org + self.domain_information_1.sub_organization = delete_org + self.domain_request_1.save() + self.domain_information_1.save() + + # But not these ones + self.domain_request_2.sub_organization = unrelated_org + self.domain_information_2.sub_organization = unrelated_org + self.domain_request_2.save() + self.domain_information_2.save() + + self.run_patch_suborganizations() + + self.domain_request_1.refresh_from_db() + self.domain_information_1.refresh_from_db() + self.domain_request_2.refresh_from_db() + self.domain_information_2.refresh_from_db() + + self.assertEqual(self.domain_request_1.sub_organization, keep_org) + self.assertEqual(self.domain_information_1.sub_organization, keep_org) + self.assertEqual(self.domain_request_2.sub_organization, unrelated_org) + self.assertEqual(self.domain_information_2.sub_organization, unrelated_org) From 5ff59efbb6ac1fc2c8ffa0ebc53f58f775aa817a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 16:41:49 -0500 Subject: [PATCH 19/58] tests for portfolio views --- src/registrar/tests/test_views_portfolio.py | 63 ++++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 18f68a7fb..490c6b1bb 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2211,7 +2211,8 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) - def test_post_with_valid_removed_domains(self): + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_with_valid_removed_domains(self, mock_send_domain_email): """Test that domains can be successfully removed.""" self.client.force_login(self.user) @@ -2233,6 +2234,8 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): messages = list(response.wsgi_request._messages) self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "The domain assignment changes have been saved.") + # assert that send_domain_invitation_email is not called + mock_send_domain_email.assert_not_called() UserDomainRole.objects.all().delete() @@ -2300,6 +2303,29 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "No changes detected.") + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @override_flag("organization_members", active=True) + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_when_send_domain_email_raises_exception(self, mock_send_domain_email): + """Test attempt to add new domains when an EmailSendingError raised.""" + self.client.force_login(self.user) + + data = { + "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs for the attempted add + } + mock_send_domain_email.side_effect = EmailSendingError("Failed to send email") + response = self.client.post(self.url, data) + + # Check that the UserDomainRole objects were not created + self.assertEqual(UserDomainRole.objects.filter(user=self.user, role=UserDomainRole.Roles.MANAGER).count(), 0) + + # Check for an error message and a redirect to edit form + self.assertRedirects(response, reverse("member-domains-edit", kwargs={"pk": self.portfolio_permission.pk})) + messages = list(response.wsgi_request._messages) + self.assertEqual(len(messages), 1) + self.assertEqual(str(messages[0]), "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.") + class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomainsView): @classmethod @@ -2412,7 +2438,7 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) @patch("registrar.views.portfolios.send_domain_invitation_email") - def test_post_with_existing_and_new_added_domains(self, mock_send_domain_email): + def test_post_with_existing_and_new_added_domains(self, _): """Test updating existing and adding new invitations.""" self.client.force_login(self.user) @@ -2452,7 +2478,8 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) - def test_post_with_valid_removed_domains(self): + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_with_valid_removed_domains(self, mock_send_domain_email): """Test removing domains successfully.""" self.client.force_login(self.user) @@ -2487,6 +2514,8 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain # Check for a success message and a redirect self.assertRedirects(response, reverse("invitedmember-domains", kwargs={"pk": self.invitation.pk})) + # assert that send_domain_invitation_email is not called + mock_send_domain_email.assert_not_called() @less_console_noise_decorator @override_flag("organization_feature", active=True) @@ -2552,6 +2581,34 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "No changes detected.") + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @override_flag("organization_members", active=True) + @patch("registrar.views.portfolios.send_domain_invitation_email") + def test_post_when_send_domain_email_raises_exception(self, mock_send_domain_email): + """Test attempt to add new domains when an EmailSendingError raised.""" + self.client.force_login(self.user) + + data = { + "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs for the attempted add + } + mock_send_domain_email.side_effect = EmailSendingError("Failed to send email") + response = self.client.post(self.url, data) + + # Check that the DomainInvitation objects were not created + self.assertEqual( + DomainInvitation.objects.filter( + email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + ).count(), + 0, + ) + + # Check for an error message and a redirect to edit form + self.assertRedirects(response, reverse("invitedmember-domains-edit", kwargs={"pk": self.invitation.pk})) + messages = list(response.wsgi_request._messages) + self.assertEqual(len(messages), 1) + self.assertEqual(str(messages[0]), "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.") + class TestRequestingEntity(WebTest): """The requesting entity page is a domain request form that only exists From 2fa8366206e40a1b99e59a1eb5ada656fb34e3c0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:46:37 -0700 Subject: [PATCH 20/58] Lint --- .../commands/create_federal_portfolio.py | 182 +++++++++--------- .../commands/patch_suborganizations.py | 3 +- .../tests/test_management_scripts.py | 3 +- 3 files changed, 91 insertions(+), 97 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 31f4b1e28..23b634a84 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -80,6 +80,22 @@ class Command(BaseCommand): else: raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") + # C901 'Command.handle' is too complex (12) + self.handle_all_populate_portfolio(agencies, parse_domains, parse_requests, both) + TerminalHelper.log_script_run_summary( + self.updated_portfolios, + self.failed_portfolios, + self.skipped_portfolios, + debug=False, + skipped_header="----- SOME PORTFOLIOS WERE SKIPPED -----", + display_as_str=True, + ) + + def handle_all_populate_portfolio(self, agencies, parse_domains, parse_requests, both): + """Loops through every agency and creates a portfolio for each. + For a given portfolio, it adds suborgs, and associates + the suborg and portfolio to domains and domain requests. + """ all_suborganizations = [] all_domains = [] all_domain_requests = [] @@ -87,13 +103,19 @@ class Command(BaseCommand): message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) try: - # C901 'Command.handle' is too complex (12) - suborgs, domains, requests = self.handle_populate_portfolio( - federal_agency, parse_domains, parse_requests, both - ) - all_suborganizations.extend(suborgs) + portfolio, created = self.create_portfolio(federal_agency) + suborganizations = self.create_suborganizations(portfolio, federal_agency) + domains = [] + domain_requests = [] + if created and parse_domains or both: + domains = self.handle_portfolio_domains(portfolio, federal_agency) + + if parse_requests or both: + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) + + all_suborganizations.extend(suborganizations) all_domains.extend(domains) - all_domain_requests.extend(requests) + all_domain_requests.extend(domain_requests) except Exception as exec: self.failed_portfolios.add(federal_agency) logger.error(exec) @@ -107,88 +129,6 @@ class Command(BaseCommand): message = f"Added city and state_territory information to {len(all_suborganizations)} suborgs." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) - TerminalHelper.log_script_run_summary( - self.updated_portfolios, - self.failed_portfolios, - self.skipped_portfolios, - debug=False, - skipped_header="----- SOME PORTFOLIOS WERE SKIPPED -----", - display_as_str=True, - ) - - def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): - """Attempts to create a portfolio. If successful, this function will - also create new suborganizations""" - portfolio, created = self.create_portfolio(federal_agency) - suborganizations = self.create_suborganizations(portfolio, federal_agency) - domains = [] - domain_requests = [] - if created: - if parse_domains or both: - domains = self.handle_portfolio_domains(portfolio, federal_agency) - - if parse_requests or both: - domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) - - return suborganizations, domains, domain_requests - - def post_process_suborganization_fields(self, suborganizations, domains, requests): - """Post-process suborganization fields by pulling data from related domains and requests. - - This function updates suborganization city and state_territory fields based on - related domain information and domain request information. - """ - - # Exclude domains and requests where the org name is the same, - # and where we are missing some crucial information. - domains = DomainInformation.objects.filter(id__in=[domain.id for domain in domains]).exclude( - portfolio__isnull=True, - organization_name__isnull=True, - sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name"), - ) - domains_dict = {domain.organization_name: domain for domain in domains} - - requests = DomainRequest.objects.filter(id__in=[request.id for request in requests]).exclude( - portfolio__isnull=True, - organization_name__isnull=True, - sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name"), - ) - requests_dict = {request.organization_name: request for request in requests} - - for suborg in suborganizations: - domain = domains_dict.get(suborg.name, None) - request = requests_dict.get(suborg.name, None) - - # PRIORITY: - # 1. Domain info - # 2. Domain request requested suborg fields - # 3. Domain request normal fields - city = None - if domain and domain.city: - city = normalize_string(domain.city, lowercase=False) - elif request and request.suborganization_city: - city = normalize_string(request.suborganization_city, lowercase=False) - elif request and request.city: - city = normalize_string(request.city, lowercase=False) - - state_territory = None - if domain and domain.state_territory: - state_territory = domain.state_territory - elif request and request.suborganization_state_territory: - state_territory = request.suborganization_state_territory - elif request and request.state_territory: - state_territory = request.state_territory - - if city: - suborg.city = city - - if suborg: - suborg.state_territory = state_territory - - Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) - def create_portfolio(self, federal_agency): """Creates a portfolio if it doesn't presently exist. Returns portfolio, created.""" @@ -278,6 +218,7 @@ class Command(BaseCommand): ) else: TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") + return new_suborgs def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): @@ -285,14 +226,14 @@ class Command(BaseCommand): Associate portfolio with domain requests for a federal agency. Updates all relevant domain request records. """ + invalid_states = [ + DomainRequest.DomainRequestStatus.STARTED, + DomainRequest.DomainRequestStatus.INELIGIBLE, + DomainRequest.DomainRequestStatus.REJECTED, + ] domain_requests = DomainRequest.objects.filter(federal_agency=federal_agency, portfolio__isnull=True).exclude( - status__in=[ - DomainRequest.DomainRequestStatus.STARTED, - DomainRequest.DomainRequestStatus.INELIGIBLE, - DomainRequest.DomainRequestStatus.REJECTED, - ] + status__in=invalid_states ) - if not domain_requests.exists(): message = f""" Portfolio '{portfolio}' not added to domain requests: no valid records found. @@ -346,3 +287,56 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) return domain_infos + + def post_process_suborganization_fields(self, suborganizations, domains, requests): + """Post-process suborganization fields by pulling data from related domains and requests. + + This function updates suborganization city and state_territory fields based on + related domain information and domain request information. + """ + domains = DomainInformation.objects.filter(id__in=[domain.id for domain in domains]).exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name"), + ) + requests = DomainRequest.objects.filter(id__in=[request.id for request in requests]).exclude( + portfolio__isnull=True, + organization_name__isnull=True, + sub_organization__isnull=True, + organization_name__iexact=F("portfolio__organization_name"), + ) + domains_dict = {domain.organization_name: domain for domain in domains} + requests_dict = {request.organization_name: request for request in requests} + + for suborg in suborganizations: + domain = domains_dict.get(suborg.name, None) + request = requests_dict.get(suborg.name, None) + + # PRIORITY: + # 1. Domain info + # 2. Domain request requested suborg fields + # 3. Domain request normal fields + city = None + if domain and domain.city: + city = normalize_string(domain.city, lowercase=False) + elif request and request.suborganization_city: + city = normalize_string(request.suborganization_city, lowercase=False) + elif request and request.city: + city = normalize_string(request.city, lowercase=False) + + state_territory = None + if domain and domain.state_territory: + state_territory = domain.state_territory + elif request and request.suborganization_state_territory: + state_territory = request.suborganization_state_territory + elif request and request.state_territory: + state_territory = request.state_territory + + if city: + suborg.city = city + + if suborg: + suborg.state_territory = state_territory + + Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index b06cbb05b..80676d6d2 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -60,7 +60,8 @@ class Command(BaseCommand): # Delete data from our preset list if normalized_name in extra_records_to_prune: # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. - # This assumes that there is only one item in the name_group array (see usda/oc example). Should be fine, given our data. + # This assumes that there is only one item in the name_group array (see usda/oc example). + # But this should be fine, given our data. hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] name_group = name_groups.get(normalize_string(hardcoded_record_name)) keep = name_group[0] if name_group else None diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 1be62ab86..0ae1f691e 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1,5 +1,4 @@ import copy -from io import StringIO import boto3_mocking # type: ignore from datetime import date, datetime, time from django.core.management import call_command @@ -1746,7 +1745,7 @@ class TestCreateFederalPortfolio(TestCase): self.domain_info.state_territory = "NY" self.domain_info.save() - self.domain_request_2.organization_name = "super" + self.domain_request.organization_name = "super" self.domain_request.suborganization_city = "Request Suborg City" self.domain_request.suborganization_state_territory = "CA" self.domain_request.city = "Request City" From bf9824a109621ad4b4e0adb72407153cdb5556e3 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:58:37 -0700 Subject: [PATCH 21/58] Lint part 2 --- .../commands/create_federal_portfolio.py | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 23b634a84..75a85dd14 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -61,26 +61,10 @@ class Command(BaseCommand): parse_domains = options.get("parse_domains") both = options.get("both") - if not both: - if not parse_requests and not parse_domains: - raise CommandError("You must specify at least one of --parse_requests or --parse_domains.") - else: - if parse_requests or parse_domains: - raise CommandError("You cannot pass --parse_requests or --parse_domains when passing --both.") - - federal_agency_filter = {"agency__iexact": agency_name} if agency_name else {"federal_type": branch} - agencies = FederalAgency.objects.filter(**federal_agency_filter) - if not agencies or agencies.count() < 1: - if agency_name: - raise CommandError( - f"Cannot find the federal agency '{agency_name}' in our database. " - "The value you enter for `agency_name` must be " - "prepopulated in the FederalAgency table before proceeding." - ) - else: - raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") - # C901 'Command.handle' is too complex (12) + self.validate_parse_options(parse_requests, parse_domains, both) + + agencies = self.get_agencies(agency_name, branch) self.handle_all_populate_portfolio(agencies, parse_domains, parse_requests, both) TerminalHelper.log_script_run_summary( self.updated_portfolios, @@ -91,6 +75,32 @@ class Command(BaseCommand): display_as_str=True, ) + def validate_parse_options(self, parse_requests, parse_domains, both): + """Validates parse options. Raises a CommandError if invalid.""" + if not both: + if not parse_requests and not parse_domains: + raise CommandError("You must specify at least one of --parse_requests or --parse_domains.") + else: + if parse_requests or parse_domains: + raise CommandError("You cannot pass --parse_requests or --parse_domains when passing --both.") + + def get_agencies(self, agency_name, branch): + """Get federal agencies based on command options. Raises a CommandError if invalid.""" + federal_agency_filter = {"agency__iexact": agency_name} if agency_name else {"federal_type": branch} + agencies = FederalAgency.objects.filter(**federal_agency_filter) + + if not agencies.exists(): + if agency_name: + raise CommandError( + f"Cannot find the federal agency '{agency_name}' in our database. " + "The value you enter for `agency_name` must be " + "prepopulated in the FederalAgency table before proceeding." + ) + else: + raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") + + return agencies + def handle_all_populate_portfolio(self, agencies, parse_domains, parse_requests, both): """Loops through every agency and creates a portfolio for each. For a given portfolio, it adds suborgs, and associates From 25cf2fe79f36ba698a42c96d2f7d0d4e8039c6ed Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:03:30 -0700 Subject: [PATCH 22/58] Revert "Lint part 2" This reverts commit bf9824a109621ad4b4e0adb72407153cdb5556e3. --- .../commands/create_federal_portfolio.py | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 75a85dd14..23b634a84 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -61,22 +61,6 @@ class Command(BaseCommand): parse_domains = options.get("parse_domains") both = options.get("both") - # C901 'Command.handle' is too complex (12) - self.validate_parse_options(parse_requests, parse_domains, both) - - agencies = self.get_agencies(agency_name, branch) - self.handle_all_populate_portfolio(agencies, parse_domains, parse_requests, both) - TerminalHelper.log_script_run_summary( - self.updated_portfolios, - self.failed_portfolios, - self.skipped_portfolios, - debug=False, - skipped_header="----- SOME PORTFOLIOS WERE SKIPPED -----", - display_as_str=True, - ) - - def validate_parse_options(self, parse_requests, parse_domains, both): - """Validates parse options. Raises a CommandError if invalid.""" if not both: if not parse_requests and not parse_domains: raise CommandError("You must specify at least one of --parse_requests or --parse_domains.") @@ -84,12 +68,9 @@ class Command(BaseCommand): if parse_requests or parse_domains: raise CommandError("You cannot pass --parse_requests or --parse_domains when passing --both.") - def get_agencies(self, agency_name, branch): - """Get federal agencies based on command options. Raises a CommandError if invalid.""" federal_agency_filter = {"agency__iexact": agency_name} if agency_name else {"federal_type": branch} agencies = FederalAgency.objects.filter(**federal_agency_filter) - - if not agencies.exists(): + if not agencies or agencies.count() < 1: if agency_name: raise CommandError( f"Cannot find the federal agency '{agency_name}' in our database. " @@ -99,7 +80,16 @@ class Command(BaseCommand): else: raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") - return agencies + # C901 'Command.handle' is too complex (12) + self.handle_all_populate_portfolio(agencies, parse_domains, parse_requests, both) + TerminalHelper.log_script_run_summary( + self.updated_portfolios, + self.failed_portfolios, + self.skipped_portfolios, + debug=False, + skipped_header="----- SOME PORTFOLIOS WERE SKIPPED -----", + display_as_str=True, + ) def handle_all_populate_portfolio(self, agencies, parse_domains, parse_requests, both): """Loops through every agency and creates a portfolio for each. From e60d1db60f95cd47118f57ac36a08c143a5a9cdf Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:10:27 -0700 Subject: [PATCH 23/58] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 23b634a84..258f23ae8 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -54,7 +54,7 @@ class Command(BaseCommand): help="Adds portfolio to both requests and domains", ) - def handle(self, **options): + def handle(self, **options): # noqa: C901 agency_name = options.get("agency_name") branch = options.get("branch") parse_requests = options.get("parse_requests") From 6df03492b3e26da56a3f87177a310315e50ced92 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 17:20:27 -0500 Subject: [PATCH 24/58] test_views_domain tests added --- src/registrar/tests/test_views_domain.py | 59 +++++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index a3e324d42..56cac91a1 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -564,6 +564,8 @@ class TestDomainManagers(TestDomainOverview): def tearDown(self): """Ensure that the user has its original permissions""" PortfolioInvitation.objects.all().delete() + UserPortfolioPermission.objects.all().delete() + User.objects.exclude(id=self.user.id).delete() super().tearDown() @less_console_noise_decorator @@ -651,21 +653,74 @@ class TestDomainManagers(TestDomainOverview): call_args = mock_send_domain_email.call_args.kwargs self.assertEqual(call_args["email"], "mayor@igorville.gov") self.assertEqual(call_args["requestor"], self.user) - self.assertEqual(call_args["domain"], self.domain) + self.assertEqual(call_args["domains"], self.domain) self.assertIsNone(call_args.get("is_member_of_different_org")) - # Assert that the PortfolioInvitation is created + # Assert that the PortfolioInvitation is created and retrieved portfolio_invitation = PortfolioInvitation.objects.filter( email="mayor@igorville.gov", portfolio=self.portfolio ).first() self.assertIsNotNone(portfolio_invitation, "Portfolio invitation should be created.") self.assertEqual(portfolio_invitation.email, "mayor@igorville.gov") self.assertEqual(portfolio_invitation.portfolio, self.portfolio) + self.assertEqual(portfolio_invitation.status, PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED) + + #Assert that the UserPortfolioPermission is created + user_portfolio_permission = UserPortfolioPermission.objects.filter( + user=self.user, portfolio=self.portfolio + ).first() + self.assertIsNotNone(user_portfolio_permission, "User portfolio permission should be created") self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() self.assertContains(success_page, "mayor@igorville.gov") + @boto3_mocking.patching + @override_flag("organization_feature", active=True) + @less_console_noise_decorator + @patch("registrar.views.domain.send_portfolio_invitation_email") + @patch("registrar.views.domain.send_domain_invitation_email") + def test_domain_user_add_form_sends_portfolio_invitation_to_new_email(self, mock_send_domain_email, mock_send_portfolio_email): + """Adding an email not associated with a user works and sends portfolio invitation.""" + add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + + add_page.form["email"] = "notauser@igorville.gov" + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + success_result = add_page.form.submit() + + self.assertEqual(success_result.status_code, 302) + self.assertEqual( + success_result["Location"], + reverse("domain-users", kwargs={"pk": self.domain.id}), + ) + + # Verify that the invitation emails were sent + mock_send_portfolio_email.assert_called_once_with( + email="notauser@igorville.gov", requestor=self.user, portfolio=self.portfolio + ) + mock_send_domain_email.assert_called_once() + call_args = mock_send_domain_email.call_args.kwargs + self.assertEqual(call_args["email"], "notauser@igorville.gov") + self.assertEqual(call_args["requestor"], self.user) + self.assertEqual(call_args["domains"], self.domain) + self.assertIsNone(call_args.get("is_member_of_different_org")) + + # Assert that the PortfolioInvitation is created + portfolio_invitation = PortfolioInvitation.objects.filter( + email="notauser@igorville.gov", portfolio=self.portfolio + ).first() + self.assertIsNotNone(portfolio_invitation, "Portfolio invitation should be created.") + self.assertEqual(portfolio_invitation.email, "notauser@igorville.gov") + self.assertEqual(portfolio_invitation.portfolio, self.portfolio) + self.assertEqual(portfolio_invitation.status, PortfolioInvitation.PortfolioInvitationStatus.INVITED) + + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + success_page = success_result.follow() + self.assertContains(success_page, "notauser@igorville.gov") + @boto3_mocking.patching @override_flag("organization_feature", active=True) @less_console_noise_decorator From 104b1f884874f897940db46fbca6214f89e9ff19 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:23:53 -0700 Subject: [PATCH 25/58] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 258f23ae8..23b634a84 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -54,7 +54,7 @@ class Command(BaseCommand): help="Adds portfolio to both requests and domains", ) - def handle(self, **options): # noqa: C901 + def handle(self, **options): agency_name = options.get("agency_name") branch = options.get("branch") parse_requests = options.get("parse_requests") From 2f75841febe401a0f91128b48ece2068798b135b Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 9 Jan 2025 18:20:48 -0500 Subject: [PATCH 26/58] admin unit tests --- src/registrar/tests/test_admin.py | 915 ++++++++++++++++++++++++++++-- 1 file changed, 871 insertions(+), 44 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 8aeb36961..ae46ba388 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -69,7 +69,7 @@ from django.contrib.sessions.backends.db import SessionStore from django.contrib.auth import get_user_model from django.contrib import messages -from unittest.mock import ANY, patch, Mock +from unittest.mock import ANY, call, patch, Mock from django.forms import ValidationError @@ -143,6 +143,7 @@ class TestDomainInvitationAdmin(TestCase): def tearDown(self): """Delete all DomainInvitation objects""" + PortfolioInvitation.objects.all().delete() DomainInvitation.objects.all().delete() DomainInformation.objects.all().delete() Portfolio.objects.all().delete() @@ -194,14 +195,255 @@ class TestDomainInvitationAdmin(TestCase): self.assertContains(response, retrieved_html, count=1) @less_console_noise_decorator + @override_flag("organization_feature", active=True) @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_save_model_user_exists(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists. + def test_add_domain_invitation_success_when_user_not_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member. + + Should send out domain and portfolio invites. + Should trigger success messages for both email sends. + Should attempt to retrieve the domain invitation. + Should attempt to retrieve the portfolio invitation.""" + + user = User.objects.create_user(email="test@example.com", username="username") + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) + + # Assert success message + mock_messages_success.assert_has_calls([ + call(request, "test@example.com has been invited to the organization: new portfolio"), + call(request, "test@example.com has been invited to the domain: example.com"), + ]) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 1) + self.assertEqual(PortfolioInvitation.objects.first().email, "test@example.com") + + # Assert invitations were retrieved + domain_invitation = DomainInvitation.objects.get(email=user.email, domain=self.domain) + portfolio_invitation = PortfolioInvitation.objects.get(email=user.email, portfolio=self.portfolio) + + self.assertEqual(domain_invitation.status, DomainInvitation.DomainInvitationStatus.RETRIEVED) + self.assertEqual(portfolio_invitation.status, PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED) + self.assertEqual(UserDomainRole.objects.count(), 1) + self.assertEqual(UserDomainRole.objects.first().user, user) + self.assertEqual(UserPortfolioPermission.objects.count(), 1) + self.assertEqual(UserPortfolioPermission.objects.first().user, user) + + @less_console_noise_decorator + @override_flag("organization_feature", active=False) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_user_not_portfolio_member_and_organization_feature_off(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and organization_feature flag is off. + + Should send out a domain invitation. + Should not send a out portfolio invitation. + Should trigger success message for the domain invitation. + Should retrieve the domain invitation. + Should not create a portfolio invitation.""" + + user = User.objects.create_user(email="test@example.com", username="username") + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain but not portfolio + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert correct invite was created + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "test@example.com has been invited to the domain: example.com" + ) + + # Assert the domain invitation was saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + # Assert the domain invitation was retrieved + domain_invitation = DomainInvitation.objects.get(email=user.email, domain=self.domain) + + self.assertEqual(domain_invitation.status, DomainInvitation.DomainInvitationStatus.RETRIEVED) + self.assertEqual(UserDomainRole.objects.count(), 1) + self.assertEqual(UserDomainRole.objects.first().user, user) + self.assertEqual(UserPortfolioPermission.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @override_flag("multiple_portfolios", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_user_not_portfolio_member_and_multiple_portfolio_feature_on(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and multiple_portfolio flag is on. + + Should send out a domain invitation. + Should not send a out portfolio invitation. + Should trigger success message for the domain invitation. + Should retrieve the domain invitation. + Should not create a portfolio invitation.""" + + user = User.objects.create_user(email="test@example.com", username="username") + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain but not portfolio + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert correct invite was created + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "test@example.com has been invited to the domain: example.com" + ) + + # Assert the domain invitation was saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + # Assert the domain invitation was retrieved + domain_invitation = DomainInvitation.objects.get(email=user.email, domain=self.domain) + + self.assertEqual(domain_invitation.status, DomainInvitation.DomainInvitationStatus.RETRIEVED) + self.assertEqual(UserDomainRole.objects.count(), 1) + self.assertEqual(UserDomainRole.objects.first().user, user) + self.assertEqual(UserPortfolioPermission.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_user_existing_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and a portfolio invitation exists. + + Should send out domain invitation only. + Should trigger success message for the domain invitation. + Should retrieve the domain invitation.""" + + user = User.objects.create_user(email="test@example.com", username="username") + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + UserPortfolioPermission.objects.create(user=user, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER]) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_not_called + + # Assert retrieve was not called + domain_invitation_mock_retrieve.assert_called_once() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "test@example.com has been invited to the domain: example.com" + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_portfolio_email(self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending portfolio email exception. + + Should only attempt to send the portfolio invitation. + Should trigger error message on portfolio invitation. + Should not attempt to retrieve the domain invitation.""" + + mock_send_portfolio_email.side_effect = MissingEmailError("craving a burger") - Should attempt to retrieve the domain invitation.""" - # Create a user with the same email User.objects.create_user(email="test@example.com", username="username") # Create a domain invitation instance @@ -213,25 +455,172 @@ class TestDomainInvitationAdmin(TestCase): request = self.factory.post("/admin/registrar/DomainInvitation/add/") request.user = self.superuser - # Patch the retrieve method - with patch.object(DomainInvitation, "retrieve") as mock_retrieve: - admin_instance.save_model(request, invitation, form=None, change=False) + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) - # Assert retrieve was called - mock_retrieve.assert_called_once() + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_not_called() + mock_send_portfolio_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) - # Assert the invitation was saved - self.assertEqual(DomainInvitation.objects.count(), 1) - self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 0) @less_console_noise_decorator + @override_flag("organization_feature", active=True) @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_save_model_user_does_not_exist(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + + Should send out the portfolio invitation and attempt to send the domain invitation. + Should trigger portfolio invitation success message. + Should trigger domain invitation error message. + Should not attempt to retrieve the domain invitation. + Should attempt to retrieve the portfolio invitation.""" + + mock_send_domain_email.side_effect = MissingEmailError("craving a burger") + + user = User.objects.create_user(email="test@example.com", username="username") + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_called_once() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "test@example.com has been invited to the organization: new portfolio" + ) + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 1) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_existing_portfolio_member_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + + Should send out the portfolio invitation and attempt to send the domain invitation. + Should trigger portfolio invitation success message. + Should trigger domain invitation error message. + Should not attempt to retrieve the domain invitation. + Should attempt to retrieve the portfolio invitation.""" + + mock_send_domain_email.side_effect = MissingEmailError("craving a burger") + + user = User.objects.create_user(email="test@example.com", username="username") + + UserPortfolioPermission.objects.create( + user=user, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + ) + + # Create a domain invitation instance + invitation = DomainInvitation(email="test@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="test@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=user, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_not_called() + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_email_not_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): """Test saving a domain invitation when the user does not exist. - Should not attempt to retrieve the domain invitation.""" + Should send out domain and portfolio invitations. + Should trigger success messages. + Should not attempt to retrieve the domain invitation. + Should not attempt to retrieve the portfolio invitation.""" # Create a domain invitation instance invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) @@ -242,15 +631,370 @@ class TestDomainInvitationAdmin(TestCase): request.user = self.superuser # Patch the retrieve method to ensure it is not called - with patch.object(DomainInvitation, "retrieve") as mock_retrieve: - admin_instance.save_model(request, invitation, form=None, change=False) + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=None, + ) + mock_send_portfolio_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) # Assert retrieve was not called - mock_retrieve.assert_not_called() + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() - # Assert the invitation was saved + # Assert success message + mock_messages_success.assert_has_calls([ + call(request, "nonexistent@example.com has been invited to the organization: new portfolio"), + call(request, "nonexistent@example.com has been invited to the domain: example.com"), + ]) + + # Assert the invitations were saved self.assertEqual(DomainInvitation.objects.count(), 1) self.assertEqual(DomainInvitation.objects.first().email, "nonexistent@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 1) + self.assertEqual(PortfolioInvitation.objects.first().email, "nonexistent@example.com") + + @less_console_noise_decorator + @override_flag("organization_feature", active=False) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_email_not_portfolio_member_and_organization_feature_off(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user does not exist and organization_feature flag is off. + + Should send out a domain invitation. + Should not send a out portfolio invitation. + Should trigger success message for domain invitation. + Should not retrieve the domain invitation. + Should not create a portfolio invitation.""" + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain but not portfolio + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=None, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "nonexistent@example.com has been invited to the domain: example.com" + ) + + # Assert the domain invitation was saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "nonexistent@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @override_flag("multiple_portfolios", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_email_not_portfolio_member_and_multiple_portfolio_feature_on(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user does not exist and multiple_portfolio flag is on. + + Should send out a domain invitation. + Should not send a out portfolio invitation. + Should trigger success message for domain invitation. + Should not retrieve the domain invitation. + Should not create a portfolio invitation.""" + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain but not portfolio + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=None, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "nonexistent@example.com has been invited to the domain: example.com" + ) + + # Assert the domain invitation was saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "nonexistent@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + def test_add_domain_invitation_success_when_email_existing_portfolio_invitation(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user does not exist and a portfolio invitation exists. + + Should send out domain invitation only. + Should trigger success message for the domain invitation. + Should not attempt to retrieve the domain invitation. + Should not attempt to retrieve the portfolio invitation.""" + + PortfolioInvitation.objects.create(email="nonexistent@example.com", portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER]) + + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=False, + requested_user=None, + ) + mock_send_portfolio_email.assert_not_called + + # Assert retrieve was not called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "nonexistent@example.com has been invited to the domain: example.com" + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 1) + self.assertEqual(DomainInvitation.objects.first().email, "nonexistent@example.com") + self.assertEqual(PortfolioInvitation.objects.count(), 1) + self.assertEqual(PortfolioInvitation.objects.first().email, "nonexistent@example.com") + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_portfolio_email(self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending portfolio email exception. + + Should only attempt to send the portfolio invitation. + Should trigger error message on portfolio invitation. + Should not attempt to retrieve the domain invitation. + Should not attempt to retrieve the portfolio invitation.""" + + mock_send_portfolio_email.side_effect = MissingEmailError("craving a burger") + + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_not_called() + mock_send_portfolio_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 0) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + + Should send out the portfolio invitation and attempt to send the domain invitation. + Should trigger portfolio invitation success message. + Should trigger domain invitation error message. + Should not attempt to retrieve the domain invitation. + Should attempt to retrieve the portfolio invitation.""" + + mock_send_domain_email.side_effect = MissingEmailError("craving a burger") + + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=None, + requested_user=None, + ) + mock_send_portfolio_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + portfolio=self.portfolio, + ) + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_called_once_with( + request, "nonexistent@example.com has been invited to the organization: new portfolio" + ) + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 1) + + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @patch("registrar.admin.send_domain_invitation_email") + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") + @patch("django.contrib.messages.error") + def test_add_domain_invitation_when_user_existing_portfolio_email_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + + Should send out the portfolio invitation and attempt to send the domain invitation. + Should trigger portfolio invitation success message. + Should trigger domain invitation error message. + Should not attempt to retrieve the domain invitation. + Should attempt to retrieve the portfolio invitation.""" + + mock_send_domain_email.side_effect = MissingEmailError("craving a burger") + + PortfolioInvitation.objects.create( + email="nonexistent@example.com", portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + ) + + # Create a domain invitation instance + invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) + + admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) + + # Create a request object + request = self.factory.post("/admin/registrar/DomainInvitation/add/") + request.user = self.superuser + + # Patch the retrieve method to ensure it is not called + with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, invitation, form=None, change=False) + + # Assert sends appropriate emails - domain and portfolio invites + mock_send_domain_email.assert_called_once_with( + email="nonexistent@example.com", + requestor=self.superuser, + domains=self.domain, + is_member_of_different_org=False, + requested_user=None, + ) + mock_send_portfolio_email.assert_not_called() + + # Assert retrieve on domain invite only was called + domain_invitation_mock_retrieve.assert_not_called() + portfolio_invitation_mock_retrieve.assert_not_called() + + # Assert success message + mock_messages_success.assert_not_called() + + # Assert error message + mock_messages_error.assert_called_once_with( + request, "Can't send invitation email. No email is associated with your user account." + ) + + # Assert the invitations were saved + self.assertEqual(DomainInvitation.objects.count(), 0) + self.assertEqual(PortfolioInvitation.objects.count(), 1) class TestUserPortfolioPermissionAdmin(TestCase): @@ -387,6 +1131,7 @@ class TestPortfolioInvitationAdmin(TestCase): Portfolio.objects.all().delete() PortfolioInvitation.objects.all().delete() Contact.objects.all().delete() + User.objects.all().delete() @classmethod def tearDownClass(self): @@ -543,34 +1288,32 @@ class TestPortfolioInvitationAdmin(TestCase): @less_console_noise_decorator def test_get_filters(self): """Ensures that our filters are displaying correctly""" - with less_console_noise(): - self.client.force_login(self.superuser) + self.client.force_login(self.superuser) - response = self.client.get( - "/admin/registrar/portfolioinvitation/", - {}, - follow=True, - ) + response = self.client.get( + "/admin/registrar/portfolioinvitation/", + {}, + follow=True, + ) - # Assert that the filters are added - self.assertContains(response, "invited", count=4) - self.assertContains(response, "Invited", count=2) - self.assertContains(response, "retrieved", count=2) - self.assertContains(response, "Retrieved", count=2) + # Assert that the filters are added + self.assertContains(response, "invited", count=4) + self.assertContains(response, "Invited", count=2) + self.assertContains(response, "retrieved", count=2) + self.assertContains(response, "Retrieved", count=2) - # Check for the HTML context specificially - invited_html = 'Invited' - retrieved_html = 'Retrieved' + # Check for the HTML context specificially + invited_html = 'Invited' + retrieved_html = 'Retrieved' - self.assertContains(response, invited_html, count=1) - self.assertContains(response, retrieved_html, count=1) + self.assertContains(response, invited_html, count=1) + self.assertContains(response, retrieved_html, count=1) @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") # Mock the `messages.warning` call - def test_save_sends_email(self, mock_messages_warning, mock_send_email): - """On save_model, an email is NOT sent if an invitation already exists.""" - self.client.force_login(self.superuser) + def test_save_sends_email(self, mock_messages_success, mock_send_email): + """On save_model, an email is sent if an invitation already exists.""" # Create an instance of the admin class admin_instance = PortfolioInvitationAdmin(PortfolioInvitation, admin_site=None) @@ -589,7 +1332,7 @@ class TestPortfolioInvitationAdmin(TestCase): # Call the save_model method admin_instance.save_model(request, portfolio_invitation, None, None) - # Assert that send_portfolio_invitation_email is not called + # Assert that send_portfolio_invitation_email is called mock_send_email.assert_called() # Get the arguments passed to send_portfolio_invitation_email @@ -601,7 +1344,7 @@ class TestPortfolioInvitationAdmin(TestCase): self.assertEqual(called_kwargs["portfolio"], self.portfolio) # Assert that a warning message was triggered - mock_messages_warning.assert_called_once_with(request, "james.gordon@gotham.gov has been invited.") + mock_messages_success.assert_called_once_with(request, "james.gordon@gotham.gov has been invited.") @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @@ -638,6 +1381,90 @@ class TestPortfolioInvitationAdmin(TestCase): # Assert that a warning message was triggered mock_messages_warning.assert_called_once_with(request, "User is already a member of this portfolio.") + @less_console_noise_decorator + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") # Mock the `messages.warning` call + def test_add_portfolio_invitation_auto_retrieves_invitation_when_user_exists(self, mock_messages_success, mock_send_email): + """On save_model, we create and retrieve a portfolio invitation if the user exists.""" + + # Create an instance of the admin class + admin_instance = PortfolioInvitationAdmin(PortfolioInvitation, admin_site=None) + + User.objects.create_user(email="james.gordon@gotham.gov", username="username") + + # Create a PortfolioInvitation instance + portfolio_invitation = PortfolioInvitation( + email="james.gordon@gotham.gov", + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + ) + + # Create a request object + request = self.factory.post("/admin/registrar/PortfolioInvitation/add/") + request.user = self.superuser + + # Call the save_model method + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, portfolio_invitation, None, None) + + # Assert that send_portfolio_invitation_email is called + mock_send_email.assert_called() + + # Get the arguments passed to send_portfolio_invitation_email + _, called_kwargs = mock_send_email.call_args + + # Assert the email content + self.assertEqual(called_kwargs["email"], "james.gordon@gotham.gov") + self.assertEqual(called_kwargs["requestor"], self.superuser) + self.assertEqual(called_kwargs["portfolio"], self.portfolio) + + # Assert that a warning message was triggered + mock_messages_success.assert_called_once_with(request, "james.gordon@gotham.gov has been invited.") + + # The invitation is not retrieved + portfolio_invitation_mock_retrieve.assert_called_once() + + @less_console_noise_decorator + @patch("registrar.admin.send_portfolio_invitation_email") + @patch("django.contrib.messages.success") # Mock the `messages.warning` call + def test_add_portfolio_invitation_does_not_retrieve_invitation_when_no_user(self, mock_messages_success, mock_send_email): + """On save_model, we create but do not retrieve a portfolio invitation if the user does not exist.""" + + # Create an instance of the admin class + admin_instance = PortfolioInvitationAdmin(PortfolioInvitation, admin_site=None) + + # Create a PortfolioInvitation instance + portfolio_invitation = PortfolioInvitation( + email="james.gordon@gotham.gov", + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + ) + + # Create a request object + request = self.factory.post("/admin/registrar/PortfolioInvitation/add/") + request.user = self.superuser + + # Call the save_model method + with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: + admin_instance.save_model(request, portfolio_invitation, None, None) + + # Assert that send_portfolio_invitation_email is called + mock_send_email.assert_called() + + # Get the arguments passed to send_portfolio_invitation_email + _, called_kwargs = mock_send_email.call_args + + # Assert the email content + self.assertEqual(called_kwargs["email"], "james.gordon@gotham.gov") + self.assertEqual(called_kwargs["requestor"], self.superuser) + self.assertEqual(called_kwargs["portfolio"], self.portfolio) + + # Assert that a warning message was triggered + mock_messages_success.assert_called_once_with(request, "james.gordon@gotham.gov has been invited.") + + # The invitation is not retrieved + portfolio_invitation_mock_retrieve.assert_not_called() + @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.error") # Mock the `messages.error` call @@ -667,7 +1494,7 @@ class TestPortfolioInvitationAdmin(TestCase): # Assert that messages.error was called with the correct message mock_messages_error.assert_called_once_with( - request, "Could not send email invitation. Portfolio invitation not saved." + request, "Email service unavailable" ) @less_console_noise_decorator @@ -705,7 +1532,7 @@ class TestPortfolioInvitationAdmin(TestCase): @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") - @patch("django.contrib.messages.error") # Mock the `messages.error` call + @patch("django.contrib.messages.warning") # Mock the `messages.error` call def test_save_exception_generic_error(self, mock_messages_error, mock_send_email): """Handle generic exceptions correctly during portfolio invitation.""" self.client.force_login(self.superuser) @@ -732,7 +1559,7 @@ class TestPortfolioInvitationAdmin(TestCase): # Assert that messages.error was called with the correct message mock_messages_error.assert_called_once_with( - request, "Could not send email invitation. Portfolio invitation not saved." + request, "Could not send email invitation." ) From ca04baeeefb6473a2fe8df36e9100f9fd60c31c3 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 18:23:24 -0500 Subject: [PATCH 27/58] test_views_portfolio tests --- src/registrar/tests/test_views_portfolio.py | 52 ++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 490c6b1bb..a6edb710d 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2958,7 +2958,7 @@ class TestPortfolioInviteNewMemberView(TestWithUser, WebTest): ], ) - cls.new_member_email = "davekenn4242@gmail.com" + cls.new_member_email = "newmember@example.com" AllowedEmail.objects.get_or_create(email=cls.new_member_email) @@ -3012,11 +3012,13 @@ class TestPortfolioInviteNewMemberView(TestWithUser, WebTest): self.assertEqual(final_response.status_code, 302) # Redirects # Validate Database Changes + # Validate that portfolio invitation was created but not retrieved portfolio_invite = PortfolioInvitation.objects.filter( email=self.new_member_email, portfolio=self.portfolio ).first() self.assertIsNotNone(portfolio_invite) self.assertEqual(portfolio_invite.email, self.new_member_email) + self.assertEqual(portfolio_invite.status, PortfolioInvitation.PortfolioInvitationStatus.INVITED) # Check that an email was sent self.assertTrue(mock_client.send_email.called) @@ -3307,6 +3309,54 @@ class TestPortfolioInviteNewMemberView(TestWithUser, WebTest): # assert that send_portfolio_invitation_email is not called mock_send_email.assert_not_called() + @less_console_noise_decorator + @override_flag("organization_feature", active=True) + @override_flag("organization_members", active=True) + @patch("registrar.views.portfolios.send_portfolio_invitation_email") + def test_member_invite_for_existing_user_who_is_not_a_member(self, mock_send_email): + """Tests the member invitation flow for existing user who is not a portfolio member.""" + self.client.force_login(self.user) + + # Simulate a session to ensure continuity + session_id = self.client.session.session_key + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + invite_count_before = PortfolioInvitation.objects.count() + + new_user = User.objects.create(email="newuser@example.com") + + # Simulate submission of member invite for the newly created user + response = self.client.post( + reverse("new-member"), + { + "role": UserPortfolioRoleChoices.ORGANIZATION_MEMBER.value, + "domain_request_permission_member": UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS.value, + "email": "newuser@example.com", + }, + ) + self.assertEqual(response.status_code, 302) + + # Validate Database Changes + # Validate that portfolio invitation was created and retrieved + portfolio_invite = PortfolioInvitation.objects.filter( + email="newuser@example.com", portfolio=self.portfolio + ).first() + self.assertIsNotNone(portfolio_invite) + self.assertEqual(portfolio_invite.email, "newuser@example.com") + self.assertEqual(portfolio_invite.status, PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED) + # Validate UserPortfolioPermission + user_portfolio_permission = UserPortfolioPermission.objects.filter( + user=new_user, portfolio=self.portfolio + ).first() + self.assertIsNotNone(user_portfolio_permission) + + # assert that send_portfolio_invitation_email is called + mock_send_email.assert_called_once() + call_args = mock_send_email.call_args.kwargs + self.assertEqual(call_args["email"], "newuser@example.com") + self.assertEqual(call_args["requestor"], self.user) + self.assertIsNone(call_args.get("is_member_of_different_org")) + class TestEditPortfolioMemberView(WebTest): """Tests for the edit member page on portfolios""" From 8f8394d2804a1c16551b5fdac1df890cf96f31fb Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 9 Jan 2025 19:19:11 -0500 Subject: [PATCH 28/58] Missing model tests on portfolio helpers, some misplaced model tests in test_admin --- src/registrar/tests/test_admin.py | 178 +--------------- src/registrar/tests/test_models.py | 325 +++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 177 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index ae46ba388..f14259dd1 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -1002,8 +1002,6 @@ class TestUserPortfolioPermissionAdmin(TestCase): def setUp(self): """Create a client object""" - self.factory = RequestFactory() - self.admin = ListHeaderAdmin(model=UserPortfolioPermissionAdmin, admin_site=AdminSite()) self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() self.portfolio = Portfolio.objects.create(organization_name="Test Portfolio", creator=self.superuser) @@ -1011,77 +1009,9 @@ class TestUserPortfolioPermissionAdmin(TestCase): def tearDown(self): """Delete all DomainInvitation objects""" Portfolio.objects.all().delete() - PortfolioInvitation.objects.all().delete() Contact.objects.all().delete() User.objects.all().delete() - - @less_console_noise_decorator - def test_clean_user_portfolio_permission(self): - """Tests validation of user portfolio permission""" - - # Test validation fails when portfolio missing but permissions are present - permission = UserPortfolioPermission(user=self.superuser, roles=["organization_admin"], portfolio=None) - with self.assertRaises(ValidationError) as err: - permission.clean() - self.assertEqual( - str(err.exception), - "When portfolio roles or additional permissions are assigned, portfolio is required.", - ) - - # Test validation fails when portfolio present but no permissions are present - permission = UserPortfolioPermission(user=self.superuser, roles=None, portfolio=self.portfolio) - with self.assertRaises(ValidationError) as err: - permission.clean() - self.assertEqual( - str(err.exception), - "When portfolio is assigned, portfolio roles or additional permissions are required.", - ) - - # Test validation fails with forbidden permissions for single role - forbidden_member_roles = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( - UserPortfolioRoleChoices.ORGANIZATION_MEMBER - ) - permission = UserPortfolioPermission( - user=self.superuser, - roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], - additional_permissions=forbidden_member_roles, - portfolio=self.portfolio, - ) - with self.assertRaises(ValidationError) as err: - permission.clean() - self.assertEqual( - str(err.exception), - "These permissions cannot be assigned to Member: " - "", - ) - - @less_console_noise_decorator - def test_get_forbidden_permissions_with_multiple_roles(self): - """Tests that forbidden permissions are properly handled when a user has multiple roles""" - # Get forbidden permissions for member role - member_forbidden = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( - UserPortfolioRoleChoices.ORGANIZATION_MEMBER - ) - - # Test with both admin and member roles - roles = [UserPortfolioRoleChoices.ORGANIZATION_ADMIN, UserPortfolioRoleChoices.ORGANIZATION_MEMBER] - - # These permissions would be forbidden for member alone, but should be allowed - # when combined with admin role - permissions = UserPortfolioPermission.get_forbidden_permissions( - roles=roles, additional_permissions=member_forbidden - ) - - # Should return empty set since no permissions are commonly forbidden between admin and member - self.assertEqual(permissions, set()) - - # Verify the same permissions are forbidden when only member role is present - member_only_permissions = UserPortfolioPermission.get_forbidden_permissions( - roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], additional_permissions=member_forbidden - ) - - # Should return the forbidden permissions for member role - self.assertEqual(member_only_permissions, set(member_forbidden)) + UserPortfolioPermission.objects.all().delete() @less_console_noise_decorator def test_has_change_form_description(self): @@ -1137,112 +1067,6 @@ class TestPortfolioInvitationAdmin(TestCase): def tearDownClass(self): User.objects.all().delete() - @less_console_noise_decorator - @override_flag("multiple_portfolios", active=False) - def test_clean_multiple_portfolios_inactive(self): - """Tests that users cannot have multiple portfolios or invitations when flag is inactive""" - # Create the first portfolio permission - UserPortfolioPermission.objects.create( - user=self.superuser, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - - # Test a second portfolio permission object (should fail) - second_portfolio = Portfolio.objects.create(organization_name="Second Portfolio", creator=self.superuser) - second_permission = UserPortfolioPermission( - user=self.superuser, portfolio=second_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - - with self.assertRaises(ValidationError) as err: - second_permission.clean() - self.assertIn("users cannot be assigned to multiple portfolios", str(err.exception)) - - # Test that adding a new portfolio invitation also fails - third_portfolio = Portfolio.objects.create(organization_name="Third Portfolio", creator=self.superuser) - invitation = PortfolioInvitation( - email=self.superuser.email, portfolio=third_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - - with self.assertRaises(ValidationError) as err: - invitation.clean() - self.assertIn("users cannot be assigned to multiple portfolios", str(err.exception)) - - @less_console_noise_decorator - @override_flag("multiple_portfolios", active=True) - def test_clean_multiple_portfolios_active(self): - """Tests that users can have multiple portfolios and invitations when flag is active""" - # Create first portfolio permission - UserPortfolioPermission.objects.create( - user=self.superuser, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - - # Second portfolio permission should succeed - second_portfolio = Portfolio.objects.create(organization_name="Second Portfolio", creator=self.superuser) - second_permission = UserPortfolioPermission( - user=self.superuser, portfolio=second_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - second_permission.clean() - second_permission.save() - - # Verify both permissions exist - user_permissions = UserPortfolioPermission.objects.filter(user=self.superuser) - self.assertEqual(user_permissions.count(), 2) - - # Portfolio invitation should also succeed - third_portfolio = Portfolio.objects.create(organization_name="Third Portfolio", creator=self.superuser) - invitation = PortfolioInvitation( - email=self.superuser.email, portfolio=third_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] - ) - invitation.clean() - invitation.save() - - # Verify invitation exists - self.assertTrue( - PortfolioInvitation.objects.filter( - email=self.superuser.email, - portfolio=third_portfolio, - ).exists() - ) - - @less_console_noise_decorator - def test_clean_portfolio_invitation(self): - """Tests validation of portfolio invitation permissions""" - - # Test validation fails when portfolio missing but permissions present - invitation = PortfolioInvitation(email="test@example.com", roles=["organization_admin"], portfolio=None) - with self.assertRaises(ValidationError) as err: - invitation.clean() - self.assertEqual( - str(err.exception), - "When portfolio roles or additional permissions are assigned, portfolio is required.", - ) - - # Test validation fails when portfolio present but no permissions - invitation = PortfolioInvitation(email="test@example.com", roles=None, portfolio=self.portfolio) - with self.assertRaises(ValidationError) as err: - invitation.clean() - self.assertEqual( - str(err.exception), - "When portfolio is assigned, portfolio roles or additional permissions are required.", - ) - - # Test validation fails with forbidden permissions - forbidden_member_roles = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( - UserPortfolioRoleChoices.ORGANIZATION_MEMBER - ) - invitation = PortfolioInvitation( - email="test@example.com", - roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], - additional_permissions=forbidden_member_roles, - portfolio=self.portfolio, - ) - with self.assertRaises(ValidationError) as err: - invitation.clean() - self.assertEqual( - str(err.exception), - "These permissions cannot be assigned to Member: " - "", - ) - @less_console_noise_decorator def test_has_model_description(self): """Tests if this model has a model description on the table view""" diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 46604a44a..31af8462a 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -28,6 +28,7 @@ from registrar.models.verified_by_staff import VerifiedByStaff # type: ignore from .common import ( MockSESClient, completed_domain_request, + create_superuser, create_test_user, ) from waffle.testutils import override_flag @@ -155,6 +156,7 @@ class TestPortfolioInvitations(TestCase): roles=[self.portfolio_role_base, self.portfolio_role_admin], additional_permissions=[self.portfolio_permission_1, self.portfolio_permission_2], ) + self.superuser = create_superuser() def tearDown(self): super().tearDown() @@ -294,10 +296,159 @@ class TestPortfolioInvitations(TestCase): # Verify self.assertEquals(self.invitation.get_portfolio_permissions(), perm_list) + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=False) + def test_clean_multiple_portfolios_inactive(self): + """Tests that users cannot have multiple portfolios or invitations when flag is inactive""" + # Create the first portfolio permission + UserPortfolioPermission.objects.create( + user=self.superuser, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + + # Test a second portfolio permission object (should fail) + second_portfolio = Portfolio.objects.create(organization_name="Second Portfolio", creator=self.superuser) + second_permission = UserPortfolioPermission( + user=self.superuser, portfolio=second_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + + with self.assertRaises(ValidationError) as err: + second_permission.clean() + self.assertIn("users cannot be assigned to multiple portfolios", str(err.exception)) + + # Test that adding a new portfolio invitation also fails + third_portfolio = Portfolio.objects.create(organization_name="Third Portfolio", creator=self.superuser) + invitation = PortfolioInvitation( + email=self.superuser.email, portfolio=third_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + + with self.assertRaises(ValidationError) as err: + invitation.clean() + self.assertIn("users cannot be assigned to multiple portfolios", str(err.exception)) + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=True) + def test_clean_multiple_portfolios_active(self): + """Tests that users can have multiple portfolios and invitations when flag is active""" + # Create first portfolio permission + UserPortfolioPermission.objects.create( + user=self.superuser, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + + # Second portfolio permission should succeed + second_portfolio = Portfolio.objects.create(organization_name="Second Portfolio", creator=self.superuser) + second_permission = UserPortfolioPermission( + user=self.superuser, portfolio=second_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + second_permission.clean() + second_permission.save() + + # Verify both permissions exist + user_permissions = UserPortfolioPermission.objects.filter(user=self.superuser) + self.assertEqual(user_permissions.count(), 2) + + # Portfolio invitation should also succeed + third_portfolio = Portfolio.objects.create(organization_name="Third Portfolio", creator=self.superuser) + invitation = PortfolioInvitation( + email=self.superuser.email, portfolio=third_portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN] + ) + invitation.clean() + invitation.save() + + # Verify invitation exists + self.assertTrue( + PortfolioInvitation.objects.filter( + email=self.superuser.email, + portfolio=third_portfolio, + ).exists() + ) + + @less_console_noise_decorator + def test_clean_portfolio_invitation(self): + """Tests validation of portfolio invitation permissions""" + + # Test validation fails when portfolio missing but permissions present + invitation = PortfolioInvitation(email="test@example.com", roles=["organization_admin"], portfolio=None) + with self.assertRaises(ValidationError) as err: + invitation.clean() + self.assertEqual( + str(err.exception), + "When portfolio roles or additional permissions are assigned, portfolio is required.", + ) + + # Test validation fails when portfolio present but no permissions + invitation = PortfolioInvitation(email="test@example.com", roles=None, portfolio=self.portfolio) + with self.assertRaises(ValidationError) as err: + invitation.clean() + self.assertEqual( + str(err.exception), + "When portfolio is assigned, portfolio roles or additional permissions are required.", + ) + + # Test validation fails with forbidden permissions + forbidden_member_roles = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( + UserPortfolioRoleChoices.ORGANIZATION_MEMBER + ) + invitation = PortfolioInvitation( + email="test@example.com", + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], + additional_permissions=forbidden_member_roles, + portfolio=self.portfolio, + ) + with self.assertRaises(ValidationError) as err: + invitation.clean() + self.assertEqual( + str(err.exception), + "These permissions cannot be assigned to Member: " + "", + ) + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=False) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_duplicate_permission(self): + """MISSING TEST: Test validation of multiple_portfolios flag. + Scenario 1: Flag is inactive, and the user has existing portfolio permissions + + NOTE: Refer to the same test under TestUserPortfolioPermission""" + + pass + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=False) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_existing_invitation(self): + """MISSING TEST: Test validation of multiple_portfolios flag. + Scenario 2: Flag is inactive, and the user has existing portfolio invitation to another portfolio + + NOTE: Refer to the same test under TestUserPortfolioPermission""" + + pass + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=True) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_duplicate_permission(self): + """MISSING TEST: Test validation of multiple_portfolios flag. + Scenario 3: Flag is active, and the user has existing portfolio invitation + + NOTE: Refer to the same test under TestUserPortfolioPermission""" + + pass + + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=True) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_existing_invitation(self): + """MISSING TEST: Test validation of multiple_portfolios flag. + Scenario 4: Flag is active, and the user has existing portfolio invitation to another portfolio + + NOTE: Refer to the same test under TestUserPortfolioPermission""" + + pass + class TestUserPortfolioPermission(TestCase): @less_console_noise_decorator def setUp(self): + self.superuser = create_superuser() + self.portfolio = Portfolio.objects.create(organization_name="Test Portfolio", creator=self.superuser) self.user, _ = User.objects.get_or_create(email="mayor@igorville.gov") self.user2, _ = User.objects.get_or_create(email="user2@igorville.gov", username="user2") super().setUp() @@ -311,6 +462,7 @@ class TestUserPortfolioPermission(TestCase): Portfolio.objects.all().delete() User.objects.all().delete() UserDomainRole.objects.all().delete() + PortfolioInvitation.objects.all().delete() @less_console_noise_decorator @override_flag("multiple_portfolios", active=True) @@ -427,6 +579,179 @@ class TestUserPortfolioPermission(TestCase): # Assert self.assertEqual(portfolio_permission.get_managed_domains_count(), 1) + @less_console_noise_decorator + def test_clean_user_portfolio_permission(self): + """Tests validation of user portfolio permission""" + + # Test validation fails when portfolio missing but permissions are present + permission = UserPortfolioPermission(user=self.superuser, roles=["organization_admin"], portfolio=None) + with self.assertRaises(ValidationError) as err: + permission.clean() + self.assertEqual( + str(err.exception), + "When portfolio roles or additional permissions are assigned, portfolio is required.", + ) + + # Test validation fails when portfolio present but no permissions are present + permission = UserPortfolioPermission(user=self.superuser, roles=None, portfolio=self.portfolio) + with self.assertRaises(ValidationError) as err: + permission.clean() + self.assertEqual( + str(err.exception), + "When portfolio is assigned, portfolio roles or additional permissions are required.", + ) + + # Test validation fails with forbidden permissions for single role + forbidden_member_roles = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( + UserPortfolioRoleChoices.ORGANIZATION_MEMBER + ) + permission = UserPortfolioPermission( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], + additional_permissions=forbidden_member_roles, + portfolio=self.portfolio, + ) + with self.assertRaises(ValidationError) as err: + permission.clean() + self.assertEqual( + str(err.exception), + "These permissions cannot be assigned to Member: " + "", + ) + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=False) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_duplicate_permission(self): + """Test validation of multiple_portfolios flag. + Scenario 1: Flag is inactive, and the user has existing portfolio permissions""" + + #existing permission + UserPortfolioPermission.objects.create( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + permission = UserPortfolioPermission( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + with self.assertRaises(ValidationError) as err: + permission.clean() + + self.assertEqual( + str(err.exception.messages[0]), + "This user is already assigned to a portfolio. " + "Based on current waffle flag settings, users cannot be assigned to multiple portfolios.", + ) + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=False) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_existing_invitation(self): + """Test validation of multiple_portfolios flag. + Scenario 2: Flag is inactive, and the user has existing portfolio invitation to another portfolio""" + + portfolio2 = Portfolio.objects.create(creator=self.superuser, organization_name="Joey go away") + + PortfolioInvitation.objects.create( + email=self.superuser.email, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], portfolio=portfolio2 + ) + + permission = UserPortfolioPermission( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + with self.assertRaises(ValidationError) as err: + permission.clean() + + self.assertEqual( + str(err.exception.messages[0]), + "This user is already assigned to a portfolio invitation. " + "Based on current waffle flag settings, users cannot be assigned to multiple portfolios.", + ) + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=True) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_duplicate_permission(self): + """Test validation of multiple_portfolios flag. + Scenario 3: Flag is active, and the user has existing portfolio invitation""" + + # existing permission + UserPortfolioPermission.objects.create( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + permission = UserPortfolioPermission( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + # Should not raise any exceptions + try: + permission.clean() + except ValidationError: + self.fail("ValidationError was raised unexpectedly when flag is active.") + + + @less_console_noise_decorator + @override_flag("multiple_portfolios", active=True) + def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_existing_invitation(self): + """Test validation of multiple_portfolios flag. + Scenario 4: Flag is active, and the user has existing portfolio invitation to another portfolio""" + + portfolio2 = Portfolio.objects.create(creator=self.superuser, organization_name="Joey go away") + + PortfolioInvitation.objects.create( + email=self.superuser.email, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], portfolio=portfolio2 + ) + + permission = UserPortfolioPermission( + user=self.superuser, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + portfolio=self.portfolio, + ) + + # Should not raise any exceptions + try: + permission.clean() + except ValidationError: + self.fail("ValidationError was raised unexpectedly when flag is active.") + + @less_console_noise_decorator + def test_get_forbidden_permissions_with_multiple_roles(self): + """Tests that forbidden permissions are properly handled when a user has multiple roles""" + # Get forbidden permissions for member role + member_forbidden = UserPortfolioPermission.FORBIDDEN_PORTFOLIO_ROLE_PERMISSIONS.get( + UserPortfolioRoleChoices.ORGANIZATION_MEMBER + ) + + # Test with both admin and member roles + roles = [UserPortfolioRoleChoices.ORGANIZATION_ADMIN, UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + + # These permissions would be forbidden for member alone, but should be allowed + # when combined with admin role + permissions = UserPortfolioPermission.get_forbidden_permissions( + roles=roles, additional_permissions=member_forbidden + ) + + # Should return empty set since no permissions are commonly forbidden between admin and member + self.assertEqual(permissions, set()) + + # Verify the same permissions are forbidden when only member role is present + member_only_permissions = UserPortfolioPermission.get_forbidden_permissions( + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], additional_permissions=member_forbidden + ) + + # Should return the forbidden permissions for member role + self.assertEqual(member_only_permissions, set(member_forbidden)) + class TestUser(TestCase): """Test actions that occur on user login, From af644254f5d6e68368f75046610c58aae6c9f317 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 9 Jan 2025 19:29:29 -0500 Subject: [PATCH 29/58] lint --- src/registrar/tests/test_admin.py | 139 +++++++++++++------- src/registrar/tests/test_models.py | 14 +- src/registrar/tests/test_views_domain.py | 8 +- src/registrar/tests/test_views_portfolio.py | 13 +- 4 files changed, 110 insertions(+), 64 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index f14259dd1..c9f7a9032 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -28,7 +28,6 @@ from registrar.admin import ( TransitionDomainAdmin, UserGroupAdmin, PortfolioAdmin, - UserPortfolioPermissionAdmin, ) from registrar.models import ( Domain, @@ -70,8 +69,6 @@ from django.contrib.auth import get_user_model from django.contrib import messages from unittest.mock import ANY, call, patch, Mock -from django.forms import ValidationError - import logging @@ -199,7 +196,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_user_not_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_user_not_portfolio_member( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user exists and is not a portfolio member. Should send out domain and portfolio invites. @@ -235,10 +234,12 @@ class TestDomainInvitationAdmin(TestCase): ) # Assert success message - mock_messages_success.assert_has_calls([ - call(request, "test@example.com has been invited to the organization: new portfolio"), - call(request, "test@example.com has been invited to the domain: example.com"), - ]) + mock_messages_success.assert_has_calls( + [ + call(request, "test@example.com has been invited to the organization: new portfolio"), + call(request, "test@example.com has been invited to the domain: example.com"), + ] + ) # Assert the invitations were saved self.assertEqual(DomainInvitation.objects.count(), 1) @@ -262,7 +263,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_user_not_portfolio_member_and_organization_feature_off(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_user_not_portfolio_member_and_organization_feature_off( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user exists and organization_feature flag is off. Should send out a domain invitation. @@ -270,7 +273,7 @@ class TestDomainInvitationAdmin(TestCase): Should trigger success message for the domain invitation. Should retrieve the domain invitation. Should not create a portfolio invitation.""" - + user = User.objects.create_user(email="test@example.com", username="username") # Create a domain invitation instance @@ -322,7 +325,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_user_not_portfolio_member_and_multiple_portfolio_feature_on(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_user_not_portfolio_member_and_multiple_portfolio_feature_on( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user exists and multiple_portfolio flag is on. Should send out a domain invitation. @@ -330,7 +335,7 @@ class TestDomainInvitationAdmin(TestCase): Should trigger success message for the domain invitation. Should retrieve the domain invitation. Should not create a portfolio invitation.""" - + user = User.objects.create_user(email="test@example.com", username="username") # Create a domain invitation instance @@ -381,7 +386,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_user_existing_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_user_existing_portfolio_member( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user exists and a portfolio invitation exists. Should send out domain invitation only. @@ -393,7 +400,9 @@ class TestDomainInvitationAdmin(TestCase): # Create a domain invitation instance invitation = DomainInvitation(email="test@example.com", domain=self.domain) - UserPortfolioPermission.objects.create(user=user, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER]) + UserPortfolioPermission.objects.create( + user=user, portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + ) admin_instance = DomainInvitationAdmin(DomainInvitation, admin_site=None) @@ -429,14 +438,17 @@ class TestDomainInvitationAdmin(TestCase): self.assertEqual(DomainInvitation.objects.count(), 1) self.assertEqual(DomainInvitation.objects.first().email, "test@example.com") self.assertEqual(PortfolioInvitation.objects.count(), 0) - + @less_console_noise_decorator @override_flag("organization_feature", active=True) @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_portfolio_email(self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending portfolio email exception. + def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_portfolio_email( + self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member raises + sending portfolio email exception. Should only attempt to send the portfolio invitation. Should trigger error message on portfolio invitation. @@ -455,7 +467,7 @@ class TestDomainInvitationAdmin(TestCase): request = self.factory.post("/admin/registrar/DomainInvitation/add/") request.user = self.superuser - # Patch the retrieve method to ensure it is not called + # Patch the retrieve method to ensure it is not called with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: admin_instance.save_model(request, invitation, form=None, change=False) @@ -487,8 +499,11 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + def test_add_domain_invitation_when_user_not_portfolio_member_raises_exception_sending_domain_email( + self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member raises + sending domain email exception. Should send out the portfolio invitation and attempt to send the domain invitation. Should trigger portfolio invitation success message. @@ -552,8 +567,11 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_existing_portfolio_member_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + def test_add_domain_invitation_when_user_existing_portfolio_member_raises_exception_sending_domain_email( + self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member raises + sending domain email exception. Should send out the portfolio invitation and attempt to send the domain invitation. Should trigger portfolio invitation success message. @@ -614,7 +632,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_email_not_portfolio_member(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_email_not_portfolio_member( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user does not exist. Should send out domain and portfolio invitations. @@ -654,10 +674,12 @@ class TestDomainInvitationAdmin(TestCase): portfolio_invitation_mock_retrieve.assert_not_called() # Assert success message - mock_messages_success.assert_has_calls([ - call(request, "nonexistent@example.com has been invited to the organization: new portfolio"), - call(request, "nonexistent@example.com has been invited to the domain: example.com"), - ]) + mock_messages_success.assert_has_calls( + [ + call(request, "nonexistent@example.com has been invited to the organization: new portfolio"), + call(request, "nonexistent@example.com has been invited to the domain: example.com"), + ] + ) # Assert the invitations were saved self.assertEqual(DomainInvitation.objects.count(), 1) @@ -670,7 +692,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_email_not_portfolio_member_and_organization_feature_off(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_email_not_portfolio_member_and_organization_feature_off( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user does not exist and organization_feature flag is off. Should send out a domain invitation. @@ -722,7 +746,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_email_not_portfolio_member_and_multiple_portfolio_feature_on(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_email_not_portfolio_member_and_multiple_portfolio_feature_on( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user does not exist and multiple_portfolio flag is on. Should send out a domain invitation. @@ -773,7 +799,9 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") - def test_add_domain_invitation_success_when_email_existing_portfolio_invitation(self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): + def test_add_domain_invitation_success_when_email_existing_portfolio_invitation( + self, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): """Test saving a domain invitation when the user does not exist and a portfolio invitation exists. Should send out domain invitation only. @@ -781,7 +809,11 @@ class TestDomainInvitationAdmin(TestCase): Should not attempt to retrieve the domain invitation. Should not attempt to retrieve the portfolio invitation.""" - PortfolioInvitation.objects.create(email="nonexistent@example.com", portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER]) + PortfolioInvitation.objects.create( + email="nonexistent@example.com", + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], + ) # Create a domain invitation instance invitation = DomainInvitation(email="nonexistent@example.com", domain=self.domain) @@ -827,8 +859,11 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_domain_invitation_email") @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_portfolio_email(self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending portfolio email exception. + def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_portfolio_email( + self, mock_messages_error, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member raises + sending portfolio email exception. Should only attempt to send the portfolio invitation. Should trigger error message on portfolio invitation. @@ -846,7 +881,7 @@ class TestDomainInvitationAdmin(TestCase): request = self.factory.post("/admin/registrar/DomainInvitation/add/") request.user = self.superuser - # Patch the retrieve method to ensure it is not called + # Patch the retrieve method to ensure it is not called with patch.object(DomainInvitation, "retrieve") as domain_invitation_mock_retrieve: with patch.object(PortfolioInvitation, "retrieve") as portfolio_invitation_mock_retrieve: admin_instance.save_model(request, invitation, form=None, change=False) @@ -878,8 +913,11 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + def test_add_domain_invitation_when_user_not_portfolio_email_raises_exception_sending_domain_email( + self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member + raises sending domain email exception. Should send out the portfolio invitation and attempt to send the domain invitation. Should trigger portfolio invitation success message. @@ -941,8 +979,11 @@ class TestDomainInvitationAdmin(TestCase): @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") @patch("django.contrib.messages.error") - def test_add_domain_invitation_when_user_existing_portfolio_email_raises_exception_sending_domain_email(self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email): - """Test saving a domain invitation when the user exists and is not a portfolio member raises sending domain email exception. + def test_add_domain_invitation_when_user_existing_portfolio_email_raises_exception_sending_domain_email( + self, mock_messages_error, mock_messages_success, mock_send_portfolio_email, mock_send_domain_email + ): + """Test saving a domain invitation when the user exists and is not a portfolio member + raises sending domain email exception. Should send out the portfolio invitation and attempt to send the domain invitation. Should trigger portfolio invitation success message. @@ -953,7 +994,9 @@ class TestDomainInvitationAdmin(TestCase): mock_send_domain_email.side_effect = MissingEmailError("craving a burger") PortfolioInvitation.objects.create( - email="nonexistent@example.com", portfolio=self.portfolio, roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER] + email="nonexistent@example.com", + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], ) # Create a domain invitation instance @@ -1208,7 +1251,9 @@ class TestPortfolioInvitationAdmin(TestCase): @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") # Mock the `messages.warning` call - def test_add_portfolio_invitation_auto_retrieves_invitation_when_user_exists(self, mock_messages_success, mock_send_email): + def test_add_portfolio_invitation_auto_retrieves_invitation_when_user_exists( + self, mock_messages_success, mock_send_email + ): """On save_model, we create and retrieve a portfolio invitation if the user exists.""" # Create an instance of the admin class @@ -1247,11 +1292,13 @@ class TestPortfolioInvitationAdmin(TestCase): # The invitation is not retrieved portfolio_invitation_mock_retrieve.assert_called_once() - + @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @patch("django.contrib.messages.success") # Mock the `messages.warning` call - def test_add_portfolio_invitation_does_not_retrieve_invitation_when_no_user(self, mock_messages_success, mock_send_email): + def test_add_portfolio_invitation_does_not_retrieve_invitation_when_no_user( + self, mock_messages_success, mock_send_email + ): """On save_model, we create but do not retrieve a portfolio invitation if the user does not exist.""" # Create an instance of the admin class @@ -1317,9 +1364,7 @@ class TestPortfolioInvitationAdmin(TestCase): admin_instance.save_model(request, portfolio_invitation, None, None) # Assert that messages.error was called with the correct message - mock_messages_error.assert_called_once_with( - request, "Email service unavailable" - ) + mock_messages_error.assert_called_once_with(request, "Email service unavailable") @less_console_noise_decorator @patch("registrar.admin.send_portfolio_invitation_email") @@ -1382,9 +1427,7 @@ class TestPortfolioInvitationAdmin(TestCase): admin_instance.save_model(request, portfolio_invitation, None, None) # Assert that messages.error was called with the correct message - mock_messages_error.assert_called_once_with( - request, "Could not send email invitation." - ) + mock_messages_error.assert_called_once_with(request, "Could not send email invitation.") class TestHostAdmin(TestCase): diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 31af8462a..d8db0f043 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -361,7 +361,7 @@ class TestPortfolioInvitations(TestCase): portfolio=third_portfolio, ).exists() ) - + @less_console_noise_decorator def test_clean_portfolio_invitation(self): """Tests validation of portfolio invitation permissions""" @@ -407,7 +407,7 @@ class TestPortfolioInvitations(TestCase): def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_duplicate_permission(self): """MISSING TEST: Test validation of multiple_portfolios flag. Scenario 1: Flag is inactive, and the user has existing portfolio permissions - + NOTE: Refer to the same test under TestUserPortfolioPermission""" pass @@ -417,7 +417,7 @@ class TestPortfolioInvitations(TestCase): def test_clean_user_portfolio_permission_multiple_portfolios_flag_off_and_existing_invitation(self): """MISSING TEST: Test validation of multiple_portfolios flag. Scenario 2: Flag is inactive, and the user has existing portfolio invitation to another portfolio - + NOTE: Refer to the same test under TestUserPortfolioPermission""" pass @@ -427,18 +427,17 @@ class TestPortfolioInvitations(TestCase): def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_duplicate_permission(self): """MISSING TEST: Test validation of multiple_portfolios flag. Scenario 3: Flag is active, and the user has existing portfolio invitation - + NOTE: Refer to the same test under TestUserPortfolioPermission""" pass - @less_console_noise_decorator @override_flag("multiple_portfolios", active=True) def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_existing_invitation(self): """MISSING TEST: Test validation of multiple_portfolios flag. Scenario 4: Flag is active, and the user has existing portfolio invitation to another portfolio - + NOTE: Refer to the same test under TestUserPortfolioPermission""" pass @@ -625,7 +624,7 @@ class TestUserPortfolioPermission(TestCase): """Test validation of multiple_portfolios flag. Scenario 1: Flag is inactive, and the user has existing portfolio permissions""" - #existing permission + # existing permission UserPortfolioPermission.objects.create( user=self.superuser, roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], @@ -699,7 +698,6 @@ class TestUserPortfolioPermission(TestCase): except ValidationError: self.fail("ValidationError was raised unexpectedly when flag is active.") - @less_console_noise_decorator @override_flag("multiple_portfolios", active=True) def test_clean_user_portfolio_permission_multiple_portfolios_flag_on_and_existing_invitation(self): diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 56cac91a1..5a1dbbf82 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -665,7 +665,7 @@ class TestDomainManagers(TestDomainOverview): self.assertEqual(portfolio_invitation.portfolio, self.portfolio) self.assertEqual(portfolio_invitation.status, PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED) - #Assert that the UserPortfolioPermission is created + # Assert that the UserPortfolioPermission is created user_portfolio_permission = UserPortfolioPermission.objects.filter( user=self.user, portfolio=self.portfolio ).first() @@ -680,7 +680,9 @@ class TestDomainManagers(TestDomainOverview): @less_console_noise_decorator @patch("registrar.views.domain.send_portfolio_invitation_email") @patch("registrar.views.domain.send_domain_invitation_email") - def test_domain_user_add_form_sends_portfolio_invitation_to_new_email(self, mock_send_domain_email, mock_send_portfolio_email): + def test_domain_user_add_form_sends_portfolio_invitation_to_new_email( + self, mock_send_domain_email, mock_send_portfolio_email + ): """Adding an email not associated with a user works and sends portfolio invitation.""" add_page = self.app.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] @@ -720,7 +722,7 @@ class TestDomainManagers(TestDomainOverview): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) success_page = success_result.follow() self.assertContains(success_page, "notauser@igorville.gov") - + @boto3_mocking.patching @override_flag("organization_feature", active=True) @less_console_noise_decorator diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index a6edb710d..fd1a1331d 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2324,7 +2324,10 @@ class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): self.assertRedirects(response, reverse("member-domains-edit", kwargs={"pk": self.portfolio_permission.pk})) messages = list(response.wsgi_request._messages) self.assertEqual(len(messages), 1) - self.assertEqual(str(messages[0]), "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.") + self.assertEqual( + str(messages[0]), + "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.", + ) class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomainsView): @@ -2433,7 +2436,6 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.assertEqual(list(call_args["domains"]), list(expected_domains)) self.assertFalse(call_args.get("is_member_of_different_org")) - @less_console_noise_decorator @override_flag("organization_feature", active=True) @override_flag("organization_members", active=True) @@ -2607,7 +2609,10 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.assertRedirects(response, reverse("invitedmember-domains-edit", kwargs={"pk": self.invitation.pk})) messages = list(response.wsgi_request._messages) self.assertEqual(len(messages), 1) - self.assertEqual(str(messages[0]), "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.") + self.assertEqual( + str(messages[0]), + "An unexpected error occurred: Failed to send email. If the issue persists, please contact help@get.gov.", + ) class TestRequestingEntity(WebTest): @@ -3321,8 +3326,6 @@ class TestPortfolioInviteNewMemberView(TestWithUser, WebTest): session_id = self.client.session.session_key self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - invite_count_before = PortfolioInvitation.objects.count() - new_user = User.objects.create(email="newuser@example.com") # Simulate submission of member invite for the newly created user From 92225b6ba2ac87088500ce5676cf00c5470bb016 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Thu, 9 Jan 2025 19:38:50 -0500 Subject: [PATCH 30/58] fix key tyo in test_domain_user_add_form_doesnt_send_portfolio_invitation_if_already_member --- src/registrar/tests/test_views_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index 5a1dbbf82..22e21bfd1 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -758,7 +758,7 @@ class TestDomainManagers(TestDomainOverview): call_args = mock_send_domain_email.call_args.kwargs self.assertEqual(call_args["email"], "mayor@igorville.gov") self.assertEqual(call_args["requestor"], self.user) - self.assertEqual(call_args["domain"], self.domain) + self.assertEqual(call_args["domains"], self.domain) self.assertIsNone(call_args.get("is_member_of_different_org")) # Assert that no PortfolioInvitation is created From 054dc47ad2a083869d46213f0116d1c2b06433a4 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 20:13:13 -0500 Subject: [PATCH 31/58] refactored some test setup --- src/registrar/tests/test_views_portfolio.py | 58 +++++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index fd1a1331d..8ea0704bb 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2106,25 +2106,73 @@ class TestPortfolioInvitedMemberDomainsView(TestWithUser, WebTest): self.assertEqual(response.status_code, 404) -class TestPortfolioMemberDomainsEditView(TestPortfolioMemberDomainsView): +class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): @classmethod def setUpClass(cls): super().setUpClass() - cls.url = reverse("member-domains-edit", kwargs={"pk": cls.portfolio_permission.pk}) + # Create Portfolio + cls.portfolio = Portfolio.objects.create(creator=cls.user, organization_name="Test Portfolio") + names = ["1.gov", "2.gov", "3.gov"] + Domain.objects.bulk_create([Domain(name=name) for name in names]) @classmethod def tearDownClass(cls): super().tearDownClass() + Portfolio.objects.all().delete() + User.objects.all().delete() + Domain.objects.all().delete() def setUp(self): super().setUp() - names = ["1.gov", "2.gov", "3.gov"] - Domain.objects.bulk_create([Domain(name=name) for name in names]) + # Create test member + self.user_member = User.objects.create( + username="test_member", + first_name="Second", + last_name="User", + email="second@example.com", + phone="8003112345", + title="Member", + ) + + # Create test user with no perms + self.user_no_perms = User.objects.create( + username="test_user_no_perms", + first_name="No", + last_name="Permissions", + email="user_no_perms@example.com", + phone="8003112345", + title="No Permissions", + ) + # Assign permissions to the user making requests + self.portfolio_permission = UserPortfolioPermission.objects.create( + user=self.user, + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + additional_permissions=[ + UserPortfolioPermissionChoices.VIEW_MEMBERS, + UserPortfolioPermissionChoices.EDIT_MEMBERS, + ], + ) + # Assign permissions to test member + self.permission = UserPortfolioPermission.objects.create( + user=self.user_member, + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + additional_permissions=[ + UserPortfolioPermissionChoices.VIEW_MEMBERS, + UserPortfolioPermissionChoices.EDIT_MEMBERS, + ], + ) + # Create url to be used in all tests + self.url = reverse("member-domains-edit", kwargs={"pk": self.portfolio_permission.pk}) def tearDown(self): super().tearDown() UserDomainRole.objects.all().delete() - Domain.objects.all().delete() + UserPortfolioPermission.objects.all().delete() + PortfolioInvitation.objects.all().delete() + Portfolio.objects.exclude(id=self.portfolio.id).delete() + User.objects.exclude(id=self.user.id).delete() @less_console_noise_decorator @override_flag("organization_feature", active=True) From 2be63abce6cd66fa93e6fcab1894ea3878fbc611 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 20:34:46 -0500 Subject: [PATCH 32/58] updates to TestPortfolioMemberDomainsEditView --- src/registrar/tests/test_views_portfolio.py | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 8ea0704bb..1bdab9215 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2112,8 +2112,10 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): super().setUpClass() # Create Portfolio cls.portfolio = Portfolio.objects.create(creator=cls.user, organization_name="Test Portfolio") - names = ["1.gov", "2.gov", "3.gov"] - Domain.objects.bulk_create([Domain(name=name) for name in names]) + # Create domains for testing + cls.domain1 = Domain.objects.create(name="1.gov") + cls.domain2 = Domain.objects.create(name="2.gov") + cls.domain3 = Domain.objects.create(name="3.gov") @classmethod def tearDownClass(cls): @@ -2133,7 +2135,6 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): phone="8003112345", title="Member", ) - # Create test user with no perms self.user_no_perms = User.objects.create( username="test_user_no_perms", @@ -2169,6 +2170,7 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): def tearDown(self): super().tearDown() UserDomainRole.objects.all().delete() + DomainInvitation.objects.all().delete() UserPortfolioPermission.objects.all().delete() PortfolioInvitation.objects.all().delete() Portfolio.objects.exclude(id=self.portfolio.id).delete() @@ -2234,7 +2236,7 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): self.client.force_login(self.user) data = { - "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs + "added_domains": json.dumps([self.domain1.id, self.domain2.id, self.domain3.id]), # Mock domain IDs } response = self.client.post(self.url, data) @@ -2247,7 +2249,7 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "The domain assignment changes have been saved.") - expected_domains = Domain.objects.filter(id__in=[1, 2, 3]) + expected_domains = [self.domain1, self.domain2, self.domain3] # Verify that the invitation email was sent mock_send_domain_email.assert_called_once() call_args = mock_send_domain_email.call_args.kwargs @@ -2265,17 +2267,17 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): self.client.force_login(self.user) # Create some UserDomainRole objects - domains = [1, 2, 3] - UserDomainRole.objects.bulk_create([UserDomainRole(domain_id=domain, user=self.user) for domain in domains]) + domains = [self.domain1, self.domain2, self.domain3] + UserDomainRole.objects.bulk_create([UserDomainRole(domain=domain, user=self.user) for domain in domains]) data = { - "removed_domains": json.dumps([1, 2]), + "removed_domains": json.dumps([self.domain1.id, self.domain2.id]), } response = self.client.post(self.url, data) # Check that the UserDomainRole objects were deleted self.assertEqual(UserDomainRole.objects.filter(user=self.user).count(), 1) - self.assertEqual(UserDomainRole.objects.filter(domain_id=3, user=self.user).count(), 1) + self.assertEqual(UserDomainRole.objects.filter(domain=self.domain3, user=self.user).count(), 1) # Check for a success message and a redirect self.assertRedirects(response, reverse("member-domains", kwargs={"pk": self.portfolio_permission.pk})) @@ -2360,7 +2362,7 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): self.client.force_login(self.user) data = { - "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs for the attempted add + "added_domains": json.dumps([self.domain1.id, self.domain2.id, self.domain3.id]), # Mock domain IDs } mock_send_domain_email.side_effect = EmailSendingError("Failed to send email") response = self.client.post(self.url, data) From bd4264d3185cf360f0e38d74550b28943f0afeb8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 20:51:00 -0500 Subject: [PATCH 33/58] updated TestPortfolioInvitedMemberEditDomainsView --- src/registrar/tests/test_views_portfolio.py | 76 ++++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 1bdab9215..02133744f 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2380,25 +2380,67 @@ class TestPortfolioMemberDomainsEditView(TestWithUser, WebTest): ) -class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomainsView): +class TestPortfolioInvitedMemberEditDomainsView(TestWithUser, WebTest): @classmethod def setUpClass(cls): super().setUpClass() - cls.url = reverse("invitedmember-domains-edit", kwargs={"pk": cls.invitation.pk}) + # Create Portfolio + cls.portfolio = Portfolio.objects.create(creator=cls.user, organization_name="Test Portfolio") + # Create domains for testing + cls.domain1 = Domain.objects.create(name="1.gov") + cls.domain2 = Domain.objects.create(name="2.gov") + cls.domain3 = Domain.objects.create(name="3.gov") + @classmethod def tearDownClass(cls): super().tearDownClass() + Portfolio.objects.all().delete() + User.objects.all().delete() + Domain.objects.all().delete() def setUp(self): super().setUp() - names = ["1.gov", "2.gov", "3.gov"] - Domain.objects.bulk_create([Domain(name=name) for name in names]) + # Add a user with no permissions + self.user_no_perms = User.objects.create( + username="test_user_no_perms", + first_name="No", + last_name="Permissions", + email="user_no_perms@example.com", + phone="8003112345", + title="No Permissions", + ) + # Add an invited member who has been invited to manage domains + self.invited_member_email = "invited@example.com" + self.invitation = PortfolioInvitation.objects.create( + email=self.invited_member_email, + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER], + additional_permissions=[ + UserPortfolioPermissionChoices.VIEW_MEMBERS, + ], + ) + + # Assign permissions to the user making requests + UserPortfolioPermission.objects.create( + user=self.user, + portfolio=self.portfolio, + roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN], + additional_permissions=[ + UserPortfolioPermissionChoices.VIEW_MEMBERS, + UserPortfolioPermissionChoices.EDIT_MEMBERS, + ], + ) + self.url = reverse("invitedmember-domains-edit", kwargs={"pk": self.invitation.pk}) def tearDown(self): super().tearDown() Domain.objects.all().delete() DomainInvitation.objects.all().delete() + UserPortfolioPermission.objects.all().delete() + PortfolioInvitation.objects.all().delete() + Portfolio.objects.exclude(id=self.portfolio.id).delete() + User.objects.exclude(id=self.user.id).delete() @less_console_noise_decorator @override_flag("organization_feature", active=True) @@ -2459,7 +2501,7 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.client.force_login(self.user) data = { - "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs + "added_domains": json.dumps([self.domain1.id, self.domain2.id, self.domain3.id]), } response = self.client.post(self.url, data) @@ -2477,7 +2519,7 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0]), "The domain assignment changes have been saved.") - expected_domains = Domain.objects.filter(id__in=[1, 2, 3]) + expected_domains = [self.domain1, self.domain2, self.domain3] # Verify that the invitation email was sent mock_send_domain_email.assert_called_once() call_args = mock_send_domain_email.call_args.kwargs @@ -2498,29 +2540,29 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain DomainInvitation.objects.bulk_create( [ DomainInvitation( - domain_id=1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.CANCELED + domain=self.domain1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.CANCELED ), DomainInvitation( - domain_id=2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED ), ] ) data = { - "added_domains": json.dumps([1, 2, 3]), + "added_domains": json.dumps([self.domain1.id, self.domain2.id, self.domain3.id]), } response = self.client.post(self.url, data) # Check that status for domain_id=1 was updated to INVITED self.assertEqual( - DomainInvitation.objects.get(domain_id=1, email="invited@example.com").status, + DomainInvitation.objects.get(domain=self.domain1, email="invited@example.com").status, DomainInvitation.DomainInvitationStatus.INVITED, ) # Check that domain_id=3 was created as INVITED self.assertTrue( DomainInvitation.objects.filter( - domain_id=3, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain3, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED ).exists() ) @@ -2539,28 +2581,28 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain DomainInvitation.objects.bulk_create( [ DomainInvitation( - domain_id=1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED ), DomainInvitation( - domain_id=2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED ), ] ) data = { - "removed_domains": json.dumps([1]), + "removed_domains": json.dumps([self.domain1.id]), } response = self.client.post(self.url, data) # Check that the status for domain_id=1 was updated to CANCELED self.assertEqual( - DomainInvitation.objects.get(domain_id=1, email="invited@example.com").status, + DomainInvitation.objects.get(domain=self.domain1, email="invited@example.com").status, DomainInvitation.DomainInvitationStatus.CANCELED, ) # Check that domain_id=2 remains INVITED self.assertEqual( - DomainInvitation.objects.get(domain_id=2, email="invited@example.com").status, + DomainInvitation.objects.get(domain=self.domain2, email="invited@example.com").status, DomainInvitation.DomainInvitationStatus.INVITED, ) @@ -2642,7 +2684,7 @@ class TestPortfolioInvitedMemberEditDomainsView(TestPortfolioInvitedMemberDomain self.client.force_login(self.user) data = { - "added_domains": json.dumps([1, 2, 3]), # Mock domain IDs for the attempted add + "added_domains": json.dumps([self.domain1.id, self.domain2.id, self.domain3.id]), } mock_send_domain_email.side_effect = EmailSendingError("Failed to send email") response = self.client.post(self.url, data) From d2b824cc842366d2e7b7533df219b937b6ba76af Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 Jan 2025 20:57:30 -0500 Subject: [PATCH 34/58] lint --- src/registrar/tests/test_views_portfolio.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_views_portfolio.py b/src/registrar/tests/test_views_portfolio.py index 02133744f..78a4dae82 100644 --- a/src/registrar/tests/test_views_portfolio.py +++ b/src/registrar/tests/test_views_portfolio.py @@ -2390,7 +2390,6 @@ class TestPortfolioInvitedMemberEditDomainsView(TestWithUser, WebTest): cls.domain1 = Domain.objects.create(name="1.gov") cls.domain2 = Domain.objects.create(name="2.gov") cls.domain3 = Domain.objects.create(name="3.gov") - @classmethod def tearDownClass(cls): @@ -2540,10 +2539,14 @@ class TestPortfolioInvitedMemberEditDomainsView(TestWithUser, WebTest): DomainInvitation.objects.bulk_create( [ DomainInvitation( - domain=self.domain1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.CANCELED + domain=self.domain1, + email="invited@example.com", + status=DomainInvitation.DomainInvitationStatus.CANCELED, ), DomainInvitation( - domain=self.domain2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain2, + email="invited@example.com", + status=DomainInvitation.DomainInvitationStatus.INVITED, ), ] ) @@ -2581,10 +2584,14 @@ class TestPortfolioInvitedMemberEditDomainsView(TestWithUser, WebTest): DomainInvitation.objects.bulk_create( [ DomainInvitation( - domain=self.domain1, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain1, + email="invited@example.com", + status=DomainInvitation.DomainInvitationStatus.INVITED, ), DomainInvitation( - domain=self.domain2, email="invited@example.com", status=DomainInvitation.DomainInvitationStatus.INVITED + domain=self.domain2, + email="invited@example.com", + status=DomainInvitation.DomainInvitationStatus.INVITED, ), ] ) From 0ddb3a2ca7cf6894ae777c901026febda6f3973f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:46:58 -0700 Subject: [PATCH 35/58] Fix linter C901 error --- .../commands/patch_suborganizations.py | 107 +++++++++--------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/src/registrar/management/commands/patch_suborganizations.py b/src/registrar/management/commands/patch_suborganizations.py index 80676d6d2..98ff1e36f 100644 --- a/src/registrar/management/commands/patch_suborganizations.py +++ b/src/registrar/management/commands/patch_suborganizations.py @@ -27,57 +27,10 @@ class Command(BaseCommand): normalize_string("USDA/ARS/NAL"): {"replace_with": "USDA, ARS, NAL"}, } - # First: Group all suborganization names by their "normalized" names (finding duplicates). - # Returns a dict that looks like this: - # { - # "amtrak": [, , ], - # "usda/oc": [], - # ...etc - # } - # - name_groups = {} - for suborg in Suborganization.objects.all(): - normalized_name = normalize_string(suborg.name) - if normalized_name not in name_groups: - name_groups[normalized_name] = [] - name_groups[normalized_name].append(suborg) - - # Second: find the record we should keep, and the records we should delete - # Returns a dict that looks like this: - # { - # "amtrak": { - # "keep": - # "delete": [, ] - # }, - # "usda/oc": { - # "keep": , - # "delete": [] - # }, - # ...etc - # } - records_to_prune = {} - for normalized_name, duplicate_suborgs in name_groups.items(): - # Delete data from our preset list - if normalized_name in extra_records_to_prune: - # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. - # This assumes that there is only one item in the name_group array (see usda/oc example). - # But this should be fine, given our data. - hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] - name_group = name_groups.get(normalize_string(hardcoded_record_name)) - keep = name_group[0] if name_group else None - records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} - # Delete duplicates (extra spaces or casing differences) - elif len(duplicate_suborgs) > 1: - # Pick the best record (fewest spaces, most leading capitals) - best_record = max( - duplicate_suborgs, - key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True)), - ) - records_to_prune[normalized_name] = { - "keep": best_record, - "delete": [s for s in duplicate_suborgs if s != best_record], - } - + # Second: loop through every Suborganization and return a dict of what to keep, and what to delete + # for each duplicate or "incorrect" record. We do this by pruning records with extra spaces or bad caps + # Note that "extra_records_to_prune" is just a manual mapping. + records_to_prune = self.get_records_to_prune(extra_records_to_prune) if len(records_to_prune) == 0: TerminalHelper.colorful_logger(logger.error, TerminalColors.FAIL, "No suborganizations to delete.") return @@ -126,3 +79,55 @@ class Command(BaseCommand): TerminalHelper.colorful_logger( logger.error, TerminalColors.FAIL, f"Failed to delete suborganizations: {str(e)}" ) + + def get_records_to_prune(self, extra_records_to_prune): + """Maps all suborgs into a dictionary with a record to keep, and an array of records to delete.""" + # First: Group all suborganization names by their "normalized" names (finding duplicates). + # Returns a dict that looks like this: + # { + # "amtrak": [, , ], + # "usda/oc": [], + # ...etc + # } + # + name_groups = {} + for suborg in Suborganization.objects.all(): + normalized_name = normalize_string(suborg.name) + name_groups.setdefault(normalized_name, []).append(suborg) + + # Second: find the record we should keep, and the records we should delete + # Returns a dict that looks like this: + # { + # "amtrak": { + # "keep": + # "delete": [, ] + # }, + # "usda/oc": { + # "keep": , + # "delete": [] + # }, + # ...etc + # } + records_to_prune = {} + for normalized_name, duplicate_suborgs in name_groups.items(): + # Delete data from our preset list + if normalized_name in extra_records_to_prune: + # The 'keep' field expects a Suborganization but we just pass in a string, so this is just a workaround. + # This assumes that there is only one item in the name_group array (see usda/oc example). + # But this should be fine, given our data. + hardcoded_record_name = extra_records_to_prune[normalized_name]["replace_with"] + name_group = name_groups.get(normalize_string(hardcoded_record_name)) + keep = name_group[0] if name_group else None + records_to_prune[normalized_name] = {"keep": keep, "delete": duplicate_suborgs} + # Delete duplicates (extra spaces or casing differences) + elif len(duplicate_suborgs) > 1: + # Pick the best record (fewest spaces, most leading capitals) + best_record = max( + duplicate_suborgs, + key=lambda suborg: (-suborg.name.count(" "), count_capitals(suborg.name, leading_only=True)), + ) + records_to_prune[normalized_name] = { + "keep": best_record, + "delete": [s for s in duplicate_suborgs if s != best_record], + } + return records_to_prune From e58d68b39288258f97d787764d7240ce5ed35908 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 12:19:39 -0700 Subject: [PATCH 36/58] add some debug logs --- .../management/commands/create_federal_portfolio.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 23b634a84..27d7e901f 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -125,8 +125,8 @@ class Command(BaseCommand): # Post process steps # Add suborg info to created or existing suborgs. if all_suborganizations: - self.post_process_suborganization_fields(all_suborganizations, all_domains, all_domain_requests) - message = f"Added city and state_territory information to {len(all_suborganizations)} suborgs." + updated_suborg_count = self.post_process_suborganization_fields(all_suborganizations, all_domains, all_domain_requests) + message = f"Added city and state_territory information to {updated_suborg_count} suborgs." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) def create_portfolio(self, federal_agency): @@ -308,10 +308,13 @@ class Command(BaseCommand): ) domains_dict = {domain.organization_name: domain for domain in domains} requests_dict = {request.organization_name: request for request in requests} + logger.info(f"domains_dict: {domains_dict}") + logger.info(f"requests_dict: {domains_dict}") for suborg in suborganizations: domain = domains_dict.get(suborg.name, None) request = requests_dict.get(suborg.name, None) + logger.info(f"suborg {suborg}: domain: {domain} , request: {request}") # PRIORITY: # 1. Domain info @@ -338,5 +341,7 @@ class Command(BaseCommand): if suborg: suborg.state_territory = state_territory + + logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state}") - Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) + return Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) From 5e697c36a10f6c65933599f37ae2aba8308a78e6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 12:42:07 -0700 Subject: [PATCH 37/58] Update create_federal_portfolio.py --- .../management/commands/create_federal_portfolio.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 27d7e901f..2f797d47e 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -87,7 +87,7 @@ class Command(BaseCommand): self.failed_portfolios, self.skipped_portfolios, debug=False, - skipped_header="----- SOME PORTFOLIOS WERE SKIPPED -----", + skipped_header="----- SOME PORTFOLIOS WERENT CREATED -----", display_as_str=True, ) @@ -258,7 +258,7 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return domain_requests + return list(domain_requests) def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): """ @@ -286,7 +286,7 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return domain_infos + return list(domain_infos) def post_process_suborganization_fields(self, suborganizations, domains, requests): """Post-process suborganization fields by pulling data from related domains and requests. @@ -342,6 +342,6 @@ class Command(BaseCommand): if suborg: suborg.state_territory = state_territory - logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state}") + logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") return Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) From 9a887d83870b726741534f06e2d2d7676cb079fe Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:14:39 -0700 Subject: [PATCH 38/58] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 2f797d47e..5528928af 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -102,25 +102,25 @@ class Command(BaseCommand): for federal_agency in agencies: message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) - try: - portfolio, created = self.create_portfolio(federal_agency) - suborganizations = self.create_suborganizations(portfolio, federal_agency) - domains = [] - domain_requests = [] - if created and parse_domains or both: - domains = self.handle_portfolio_domains(portfolio, federal_agency) + # try: + portfolio, created = self.create_portfolio(federal_agency) + suborganizations = self.create_suborganizations(portfolio, federal_agency) + domains = [] + domain_requests = [] + if created and parse_domains or both: + domains = self.handle_portfolio_domains(portfolio, federal_agency) - if parse_requests or both: - domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) + if parse_requests or both: + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) - all_suborganizations.extend(suborganizations) - all_domains.extend(domains) - all_domain_requests.extend(domain_requests) - except Exception as exec: - self.failed_portfolios.add(federal_agency) - logger.error(exec) - message = f"Failed to create portfolio '{federal_agency.agency}'" - TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) + all_suborganizations.extend(suborganizations) + all_domains.extend(domains) + all_domain_requests.extend(domain_requests) + # except Exception as exec: + # self.failed_portfolios.add(federal_agency) + # logger.error(exec) + # message = f"Failed to create portfolio '{federal_agency.agency}'" + # TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) # Post process steps # Add suborg info to created or existing suborgs. From 19e4dff7f9c258b6ad552d38a9f6babf08dbc964 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:24:12 -0700 Subject: [PATCH 39/58] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 5528928af..562a66da7 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -258,7 +258,7 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return list(domain_requests) + return list(domain_requests) if len(domain_requests) > 0 else [] def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): """ @@ -286,7 +286,7 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return list(domain_infos) + return list(domain_infos) if len(domain_infos) > 0 else [] def post_process_suborganization_fields(self, suborganizations, domains, requests): """Post-process suborganization fields by pulling data from related domains and requests. From ac08b17f94244722ae3f0aab5a8433ebdab01768 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:55:25 -0700 Subject: [PATCH 40/58] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 562a66da7..b70df8ee4 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -102,25 +102,25 @@ class Command(BaseCommand): for federal_agency in agencies: message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) - # try: - portfolio, created = self.create_portfolio(federal_agency) - suborganizations = self.create_suborganizations(portfolio, federal_agency) - domains = [] - domain_requests = [] - if created and parse_domains or both: - domains = self.handle_portfolio_domains(portfolio, federal_agency) + try: + portfolio, created = self.create_portfolio(federal_agency) + suborganizations = self.create_suborganizations(portfolio, federal_agency) + domains = [] + domain_requests = [] + if created and parse_domains or both: + domains = self.handle_portfolio_domains(portfolio, federal_agency) - if parse_requests or both: - domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) + if parse_requests or both: + domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) - all_suborganizations.extend(suborganizations) - all_domains.extend(domains) - all_domain_requests.extend(domain_requests) - # except Exception as exec: - # self.failed_portfolios.add(federal_agency) - # logger.error(exec) - # message = f"Failed to create portfolio '{federal_agency.agency}'" - # TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) + all_suborganizations.extend(suborganizations) + all_domains.extend(domains) + all_domain_requests.extend(domain_requests) + except Exception as exec: + self.failed_portfolios.add(federal_agency) + logger.error(exec) + message = f"Failed to create portfolio '{federal_agency.agency}'" + TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) # Post process steps # Add suborg info to created or existing suborgs. @@ -179,7 +179,6 @@ class Command(BaseCommand): federal_agency=federal_agency, organization_name__isnull=False ) org_names = set(valid_agencies.values_list("organization_name", flat=True)) - if not org_names: message = ( "Could not add any suborganizations." @@ -219,7 +218,7 @@ class Command(BaseCommand): else: TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") - return new_suborgs + return new_suborgs if len(new_suborgs) > 0 else [] def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): """ From 2ca7cd468f8cfe047e28a4fc65238c939943bb67 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:03:44 -0700 Subject: [PATCH 41/58] Modify terminal helper --- .../management/commands/create_federal_portfolio.py | 9 ++++++--- .../management/commands/utility/terminal_helper.py | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index b70df8ee4..7387f076c 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -113,9 +113,12 @@ class Command(BaseCommand): if parse_requests or both: domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) - all_suborganizations.extend(suborganizations) - all_domains.extend(domains) - all_domain_requests.extend(domain_requests) + if suborganizations: + all_suborganizations.extend(suborganizations) + if all_domains: + all_domains.extend(domains) + if domain_requests: + all_domain_requests.extend(domain_requests) except Exception as exec: self.failed_portfolios.add(federal_agency) logger.error(exec) diff --git a/src/registrar/management/commands/utility/terminal_helper.py b/src/registrar/management/commands/utility/terminal_helper.py index b16ca72f2..87d9f12e5 100644 --- a/src/registrar/management/commands/utility/terminal_helper.py +++ b/src/registrar/management/commands/utility/terminal_helper.py @@ -442,13 +442,14 @@ class TerminalHelper: f.write(file_contents) @staticmethod - def colorful_logger(log_level, color, message): + def colorful_logger(log_level, color, message, exc_info=True): """Adds some color to your log output. Args: log_level: str | Logger.method -> Desired log level. ex: logger.info or "INFO" color: str | TerminalColors -> Output color. ex: TerminalColors.YELLOW or "YELLOW" message: str -> Message to display. + exc_info: bool -> Whether the log should print exc_info or not """ if isinstance(log_level, str) and hasattr(logger, log_level.lower()): @@ -462,4 +463,4 @@ class TerminalHelper: terminal_color = color colored_message = f"{terminal_color}{message}{TerminalColors.ENDC}" - log_method(colored_message) + log_method(colored_message, exc_info=exc_info) From 2d509cfe6205b661640a8a83d9c64a6727596ebb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:59:54 -0700 Subject: [PATCH 42/58] Fix sneaky bug why --- .../commands/create_federal_portfolio.py | 73 ++++++++----------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 7387f076c..f3d638ecf 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -6,6 +6,7 @@ from django.core.management import BaseCommand, CommandError from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User from django.db.models import F +from django.db.models import Q from registrar.models.utility.generic_helper import normalize_string @@ -96,29 +97,18 @@ class Command(BaseCommand): For a given portfolio, it adds suborgs, and associates the suborg and portfolio to domains and domain requests. """ - all_suborganizations = [] - all_domains = [] - all_domain_requests = [] for federal_agency in agencies: message = f"Processing federal agency '{federal_agency.agency}'..." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) try: portfolio, created = self.create_portfolio(federal_agency) - suborganizations = self.create_suborganizations(portfolio, federal_agency) - domains = [] - domain_requests = [] + self.create_suborganizations(portfolio, federal_agency) if created and parse_domains or both: - domains = self.handle_portfolio_domains(portfolio, federal_agency) + self.handle_portfolio_domains(portfolio, federal_agency) if parse_requests or both: - domain_requests = self.handle_portfolio_requests(portfolio, federal_agency) + self.handle_portfolio_requests(portfolio, federal_agency) - if suborganizations: - all_suborganizations.extend(suborganizations) - if all_domains: - all_domains.extend(domains) - if domain_requests: - all_domain_requests.extend(domain_requests) except Exception as exec: self.failed_portfolios.add(federal_agency) logger.error(exec) @@ -126,11 +116,10 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) # Post process steps - # Add suborg info to created or existing suborgs. - if all_suborganizations: - updated_suborg_count = self.post_process_suborganization_fields(all_suborganizations, all_domains, all_domain_requests) - message = f"Added city and state_territory information to {updated_suborg_count} suborgs." - TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + # Add additional suborg info where applicable. + updated_suborg_count = self.post_process_suborganization_fields(agencies) + message = f"Added city and state_territory information to {updated_suborg_count} suborgs." + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) def create_portfolio(self, federal_agency): """Creates a portfolio if it doesn't presently exist. @@ -221,8 +210,6 @@ class Command(BaseCommand): else: TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, "No suborganizations added") - return new_suborgs if len(new_suborgs) > 0 else [] - def handle_portfolio_requests(self, portfolio: Portfolio, federal_agency: FederalAgency): """ Associate portfolio with domain requests for a federal agency. @@ -260,8 +247,6 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_requests)} domain requests." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return list(domain_requests) if len(domain_requests) > 0 else [] - def handle_portfolio_domains(self, portfolio: Portfolio, federal_agency: FederalAgency): """ Associate portfolio with domains for a federal agency. @@ -288,35 +273,39 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - return list(domain_infos) if len(domain_infos) > 0 else [] - - def post_process_suborganization_fields(self, suborganizations, domains, requests): + def post_process_suborganization_fields(self, agencies): """Post-process suborganization fields by pulling data from related domains and requests. This function updates suborganization city and state_territory fields based on related domain information and domain request information. """ - domains = DomainInformation.objects.filter(id__in=[domain.id for domain in domains]).exclude( - portfolio__isnull=True, - organization_name__isnull=True, - sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name"), + # Assuming that org name, portfolio, and suborg all aren't null + # we assume that we want to add suborg info. + # as long as the org name doesnt match the portfolio name (as that implies it is the portfolio). + should_add_suborgs_filter = Q( + organization_name__isnull=False, + portfolio__isnull=False, + sub_organization__isnull=False, + ) & ~Q(organization_name__iexact=F("portfolio__organization_name")) + domains = DomainInformation.objects.filter( + should_add_suborgs_filter, + federal_agency__in=agencies, + portfolio__isnull=False ) - requests = DomainRequest.objects.filter(id__in=[request.id for request in requests]).exclude( - portfolio__isnull=True, - organization_name__isnull=True, - sub_organization__isnull=True, - organization_name__iexact=F("portfolio__organization_name"), + requests = DomainRequest.objects.filter( + should_add_suborgs_filter, + federal_agency__in=agencies, + portfolio__isnull=False ) domains_dict = {domain.organization_name: domain for domain in domains} requests_dict = {request.organization_name: request for request in requests} - logger.info(f"domains_dict: {domains_dict}") - logger.info(f"requests_dict: {domains_dict}") - - for suborg in suborganizations: + suborgs_to_edit = Suborganization.objects.filter( + Q(id__in=domains.values_list("sub_organization", flat=True)) | + Q(id__in=requests.values_list("sub_organization", flat=True)) + ) + for suborg in suborgs_to_edit: domain = domains_dict.get(suborg.name, None) request = requests_dict.get(suborg.name, None) - logger.info(f"suborg {suborg}: domain: {domain} , request: {request}") # PRIORITY: # 1. Domain info @@ -346,4 +335,4 @@ class Command(BaseCommand): logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") - return Suborganization.objects.bulk_update(suborganizations, ["city", "state_territory"]) + return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) From 51105eb8122dc0312093e06693467ac875137d45 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:08:43 -0700 Subject: [PATCH 43/58] Remove unrelated changes --- .../commands/create_federal_portfolio.py | 70 +++++++++---------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index f3d638ecf..3657dcfd6 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -82,7 +82,23 @@ class Command(BaseCommand): raise CommandError(f"Cannot find '{branch}' federal agencies in our database.") # C901 'Command.handle' is too complex (12) - self.handle_all_populate_portfolio(agencies, parse_domains, parse_requests, both) + for federal_agency in agencies: + message = f"Processing federal agency '{federal_agency.agency}'..." + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + try: + # C901 'Command.handle' is too complex (12) + self.handle_populate_portfolio(federal_agency, parse_domains, parse_requests, both) + except Exception as exec: + self.failed_portfolios.add(federal_agency) + logger.error(exec) + message = f"Failed to create portfolio '{federal_agency.agency}'" + TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) + + # POST PROCESS STEP: Add additional suborg info where applicable. + updated_suborg_count = self.post_process_suborganization_fields(agencies) + message = f"Added city and state_territory information to {updated_suborg_count} suborgs." + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + TerminalHelper.log_script_run_summary( self.updated_portfolios, self.failed_portfolios, @@ -92,34 +108,16 @@ class Command(BaseCommand): display_as_str=True, ) - def handle_all_populate_portfolio(self, agencies, parse_domains, parse_requests, both): - """Loops through every agency and creates a portfolio for each. - For a given portfolio, it adds suborgs, and associates - the suborg and portfolio to domains and domain requests. - """ - for federal_agency in agencies: - message = f"Processing federal agency '{federal_agency.agency}'..." - TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) - try: - portfolio, created = self.create_portfolio(federal_agency) - self.create_suborganizations(portfolio, federal_agency) - if created and parse_domains or both: - self.handle_portfolio_domains(portfolio, federal_agency) + def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): + """Attempts to create a portfolio. If successful, this function will + also create new suborganizations""" + portfolio, created = self.create_portfolio(federal_agency) + self.create_suborganizations(portfolio, federal_agency) + if created and parse_domains or both: + self.handle_portfolio_domains(portfolio, federal_agency) - if parse_requests or both: - self.handle_portfolio_requests(portfolio, federal_agency) - - except Exception as exec: - self.failed_portfolios.add(federal_agency) - logger.error(exec) - message = f"Failed to create portfolio '{federal_agency.agency}'" - TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) - - # Post process steps - # Add additional suborg info where applicable. - updated_suborg_count = self.post_process_suborganization_fields(agencies) - message = f"Added city and state_territory information to {updated_suborg_count} suborgs." - TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + if parse_requests or both: + self.handle_portfolio_requests(portfolio, federal_agency) def create_portfolio(self, federal_agency): """Creates a portfolio if it doesn't presently exist. @@ -280,7 +278,7 @@ class Command(BaseCommand): related domain information and domain request information. """ # Assuming that org name, portfolio, and suborg all aren't null - # we assume that we want to add suborg info. + # we assume that we want to add suborg info. # as long as the org name doesnt match the portfolio name (as that implies it is the portfolio). should_add_suborgs_filter = Q( organization_name__isnull=False, @@ -288,20 +286,16 @@ class Command(BaseCommand): sub_organization__isnull=False, ) & ~Q(organization_name__iexact=F("portfolio__organization_name")) domains = DomainInformation.objects.filter( - should_add_suborgs_filter, - federal_agency__in=agencies, - portfolio__isnull=False + should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False ) requests = DomainRequest.objects.filter( - should_add_suborgs_filter, - federal_agency__in=agencies, - portfolio__isnull=False + should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False ) domains_dict = {domain.organization_name: domain for domain in domains} requests_dict = {request.organization_name: request for request in requests} suborgs_to_edit = Suborganization.objects.filter( - Q(id__in=domains.values_list("sub_organization", flat=True)) | - Q(id__in=requests.values_list("sub_organization", flat=True)) + Q(id__in=domains.values_list("sub_organization", flat=True)) + | Q(id__in=requests.values_list("sub_organization", flat=True)) ) for suborg in suborgs_to_edit: domain = domains_dict.get(suborg.name, None) @@ -332,7 +326,7 @@ class Command(BaseCommand): if suborg: suborg.state_territory = state_territory - + logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) From 48f470872867480fc3db653d1b1d0f3f5312469e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:16:29 -0700 Subject: [PATCH 44/58] Update src/registrar/management/commands/create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 3657dcfd6..6c5b5fac2 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -5,8 +5,7 @@ import logging from django.core.management import BaseCommand, CommandError from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.models import DomainInformation, DomainRequest, FederalAgency, Suborganization, Portfolio, User -from django.db.models import F -from django.db.models import Q +from django.db.models import F, Q from registrar.models.utility.generic_helper import normalize_string From e95bcb4114c76107e0c4e7978a5c7aeb9ba979e2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:35:32 -0700 Subject: [PATCH 45/58] Remove redundant check on created --- src/registrar/management/commands/create_federal_portfolio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 6c5b5fac2..22235245a 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -110,9 +110,9 @@ class Command(BaseCommand): def handle_populate_portfolio(self, federal_agency, parse_domains, parse_requests, both): """Attempts to create a portfolio. If successful, this function will also create new suborganizations""" - portfolio, created = self.create_portfolio(federal_agency) + portfolio, _ = self.create_portfolio(federal_agency) self.create_suborganizations(portfolio, federal_agency) - if created and parse_domains or both: + if parse_domains or both: self.handle_portfolio_domains(portfolio, federal_agency) if parse_requests or both: From e184149c6c11ac03612521cf8532d453c001ffa7 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:30:47 -0700 Subject: [PATCH 46/58] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index b604aeca7..426a3ea23 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -8,7 +8,6 @@ from registrar.models import DomainInformation, DomainRequest, FederalAgency, Su from registrar.models.utility.generic_helper import normalize_string from django.db.models import F, Q -from registrar.models.utility.generic_helper import normalize_string logger = logging.getLogger(__name__) From de07b109c216512485365d8dd736d4783926a00a Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Tue, 14 Jan 2025 12:15:59 -0600 Subject: [PATCH 47/58] fix auditlog errors --- src/registrar/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 849cb6100..f13437af1 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2782,8 +2782,10 @@ class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): try: # Retrieve and order audit log entries by timestamp in descending order - audit_log_entries = LogEntry.objects.filter(object_id=object_id).order_by("-timestamp") - + audit_log_entries = LogEntry.objects.filter( + object_id=object_id, content_type__model="domainrequest" + ).order_by("-timestamp") + # Process each log entry to filter based on the change criteria for log_entry in audit_log_entries: entry = self.process_log_entry(log_entry) From 0c8a0ad1d805868a7317c56406ec7fa129c1e5d0 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Tue, 14 Jan 2025 14:35:11 -0600 Subject: [PATCH 48/58] linter fixes --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index f13437af1..bb42b66c6 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -2785,7 +2785,7 @@ class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): audit_log_entries = LogEntry.objects.filter( object_id=object_id, content_type__model="domainrequest" ).order_by("-timestamp") - + # Process each log entry to filter based on the change criteria for log_entry in audit_log_entries: entry = self.process_log_entry(log_entry) From b5970ecb373f9d8f1f779d5670f999f507402e62 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 06:34:46 -0500 Subject: [PATCH 49/58] updates in response to PR suggestions --- src/registrar/admin.py | 3 +- .../templates/emails/domain_invitation.txt | 10 +---- src/registrar/tests/test_admin.py | 5 ++- src/registrar/utility/email_invitations.py | 5 ++- src/registrar/views/portfolios.py | 8 ++-- .../views/utility/invitation_helper.py | 39 +++++++++---------- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index cdcc0400e..eb4e1737a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1520,7 +1520,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): """ if not change: domain = obj.domain - domain_org = domain.domain_info.portfolio + domain_org = getattr(domain.domain_info, "portfolio", None) requested_email = obj.email # Look up a user with that email requested_user = get_requested_user(requested_email) @@ -1536,6 +1536,7 @@ class DomainInvitationAdmin(BaseInvitationAdmin): and not flag_is_active(request, "multiple_portfolios") and domain_org is not None and not member_of_this_org + and not member_of_a_different_org ): send_portfolio_invitation_email(email=requested_email, requestor=requestor, portfolio=domain_org) portfolio_invitation, _ = PortfolioInvitation.objects.get_or_create( diff --git a/src/registrar/templates/emails/domain_invitation.txt b/src/registrar/templates/emails/domain_invitation.txt index 4959f7c23..a077bff26 100644 --- a/src/registrar/templates/emails/domain_invitation.txt +++ b/src/registrar/templates/emails/domain_invitation.txt @@ -1,15 +1,9 @@ {% autoescape off %}{# In a text file, we don't want to have HTML entities escaped #} -{% if requested_user and requested_user.first_name %} -Hi, {{ requested_user.first_name }}. -{% else %} -Hi, -{% endif %} +Hi,{% if requested_user and requested_user.first_name %} {{ requested_user.first_name }}.{% endif %} {{ requestor_email }} has invited you to manage: -{% for domain in domains %} -{{ domain.name }} +{% for domain in domains %}{{ domain.name }} {% endfor %} - To manage domain information, visit the .gov registrar . ---------------------------------------------------------------- diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index c9f7a9032..210a1a8e6 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -334,7 +334,10 @@ class TestDomainInvitationAdmin(TestCase): Should not send a out portfolio invitation. Should trigger success message for the domain invitation. Should retrieve the domain invitation. - Should not create a portfolio invitation.""" + Should not create a portfolio invitation. + + NOTE: This test may need to be reworked when the multiple_portfolio flag is fully fleshed out. + """ user = User.objects.create_user(email="test@example.com", username="username") diff --git a/src/registrar/utility/email_invitations.py b/src/registrar/utility/email_invitations.py index 1491b65a5..48c796340 100644 --- a/src/registrar/utility/email_invitations.py +++ b/src/registrar/utility/email_invitations.py @@ -48,7 +48,10 @@ def normalize_domains(domains): def get_requestor_email(requestor, domains): - """Get the requestor's email or raise an error if it's missing.""" + """Get the requestor's email or raise an error if it's missing. + + If the requestor is staff, default email is returned. + """ if requestor.is_staff: return settings.DEFAULT_FROM_EMAIL diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index b9249ca6d..c4f60ca35 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -262,7 +262,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V "A database error occurred while saving changes. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error("A database error occurred while saving changes.") + logger.error("A database error occurred while saving changes.", exc_info=True) return redirect(reverse("member-domains-edit", kwargs={"pk": pk})) except Exception as e: messages.error( @@ -270,7 +270,7 @@ class PortfolioMemberDomainsEditView(PortfolioMemberDomainsEditPermissionView, V f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error(f"An unexpected error occurred: {str(e)}") + logger.error(f"An unexpected error occurred: {str(e)}", exc_info=True) return redirect(reverse("member-domains-edit", kwargs={"pk": pk})) else: messages.info(request, "No changes detected.") @@ -479,7 +479,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission "A database error occurred while saving changes. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error("A database error occurred while saving changes.") + logger.error("A database error occurred while saving changes.", exc_info=True) return redirect(reverse("invitedmember-domains-edit", kwargs={"pk": pk})) except Exception as e: messages.error( @@ -487,7 +487,7 @@ class PortfolioInvitedMemberDomainsEditView(PortfolioMemberDomainsEditPermission f"An unexpected error occurred: {str(e)}. If the issue persists, " f"please contact {DefaultUserValues.HELP_EMAIL}.", ) - logger.error(f"An unexpected error occurred: {str(e)}.") + logger.error(f"An unexpected error occurred: {str(e)}.", exc_info=True) return redirect(reverse("invitedmember-domains-edit", kwargs={"pk": pk})) else: messages.info(request, "No changes detected.") diff --git a/src/registrar/views/utility/invitation_helper.py b/src/registrar/views/utility/invitation_helper.py index 771300406..5c730d0c3 100644 --- a/src/registrar/views/utility/invitation_helper.py +++ b/src/registrar/views/utility/invitation_helper.py @@ -1,8 +1,6 @@ from django.contrib import messages from django.db import IntegrityError -from registrar.models.portfolio_invitation import PortfolioInvitation -from registrar.models.user import User -from registrar.models.user_portfolio_permission import UserPortfolioPermission +from registrar.models import PortfolioInvitation, User, UserPortfolioPermission from registrar.utility.email import EmailSendingError import logging @@ -20,32 +18,33 @@ logger = logging.getLogger(__name__) # any view, and were initially developed for domain.py, portfolios.py and admin.py -def get_org_membership(requestor_org, requested_email, requested_user): +def get_org_membership(org, email, user): """ - Verifies if an email belongs to a different organization as a member or invited member. - Verifies if an email belongs to this organization as a member or invited member. - User does not belong to any org can be deduced from the tuple returned. + Determines if an email/user belongs to a different organization or this organization + as either a member or an invited member. - Returns a tuple (member_of_a_different_org, member_of_this_org). + This function returns a tuple (member_of_a_different_org, member_of_this_org), + which provides: + - member_of_a_different_org: True if the user/email is associated with an organization other than the given org. + - member_of_this_org: True if the user/email is associated with the given org. + + Note: This implementation assumes single portfolio ownership for a user. + If the "multiple portfolios" feature is enabled, this logic may not account for + situations where a user or email belongs to multiple organizations. """ - # COMMENT: this code does not take into account when multiple portfolios flag is set to true - - # COMMENT: shouldn't this code be based on the organization of the domain, not the org - # of the requestor? requestor could have multiple portfolios - - # Check for existing permissions or invitations for the requested user - existing_org_permission = UserPortfolioPermission.objects.filter(user=requested_user).first() - existing_org_invitation = PortfolioInvitation.objects.filter(email=requested_email).first() + # Check for existing permissions or invitations for the user + existing_org_permission = UserPortfolioPermission.objects.filter(user=user).first() + existing_org_invitation = PortfolioInvitation.objects.filter(email=email).first() # Determine membership in a different organization - member_of_a_different_org = (existing_org_permission and existing_org_permission.portfolio != requestor_org) or ( - existing_org_invitation and existing_org_invitation.portfolio != requestor_org + member_of_a_different_org = (existing_org_permission and existing_org_permission.portfolio != org) or ( + existing_org_invitation and existing_org_invitation.portfolio != org ) # Determine membership in the same organization - member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == requestor_org) or ( - existing_org_invitation and existing_org_invitation.portfolio == requestor_org + member_of_this_org = (existing_org_permission and existing_org_permission.portfolio == org) or ( + existing_org_invitation and existing_org_invitation.portfolio == org ) return member_of_a_different_org, member_of_this_org From f196dce6077c8c420c8cd517be28596327a74932 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 07:08:22 -0500 Subject: [PATCH 50/58] lint --- src/registrar/tests/test_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 210a1a8e6..2a7a52a13 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -335,7 +335,7 @@ class TestDomainInvitationAdmin(TestCase): Should trigger success message for the domain invitation. Should retrieve the domain invitation. Should not create a portfolio invitation. - + NOTE: This test may need to be reworked when the multiple_portfolio flag is fully fleshed out. """ From ac3286d7801fd165500f3b09415ed7153ae0c21c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Jan 2025 07:26:01 -0500 Subject: [PATCH 51/58] fixed bug in portfolio invitation change view query --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index eb4e1737a..2e1b15dfb 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1594,7 +1594,7 @@ class PortfolioInvitationAdmin(BaseInvitationAdmin): # Search search_fields = [ "email", - "portfolio__name", + "portfolio__organization_name", ] # Filters From c0ce6561927b76b99ea6b154d8b9adbc74fdbf1d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:56:50 -0700 Subject: [PATCH 52/58] Account for duplicates --- .../commands/create_federal_portfolio.py | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 426a3ea23..0b2fd0887 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -375,15 +375,67 @@ class Command(BaseCommand): requests = DomainRequest.objects.filter( should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False ) - domains_dict = {domain.organization_name: domain for domain in domains} - requests_dict = {request.organization_name: request for request in requests} + + # Since domains / requests can share an org name, lets first create lists of duplicate names. + # If only one item exists in this list, we can just pull that info. + # If more than one exists, then we can determine what value we should choose. + domains_dict = {} + for domain in domains: + normalized_name = normalize_string(domain.organization_name) + domains_dict.setdefault(normalized_name, []).append(domain) + + requests_dict = {} + for request in requests: + normalized_name = normalize_string(request.organization_name) + requests_dict.setdefault(normalized_name, []).append(request) + suborgs_to_edit = Suborganization.objects.filter( Q(id__in=domains.values_list("sub_organization", flat=True)) | Q(id__in=requests.values_list("sub_organization", flat=True)) ) for suborg in suborgs_to_edit: - domain = domains_dict.get(suborg.name, None) - request = requests_dict.get(suborg.name, None) + normalized_suborg_name = normalize_string(suborg.name) + domains = domains_dict.get(normalized_suborg_name, []) + requests = requests_dict.get(normalized_suborg_name, []) + + domains_length = len(domains) + if domains_length == 0: + domain = None + elif domains_length == 1: + domain = domains[0] + else: + logger.info(f"in this loop (domains): {domains}") + reference = domains[0] + use_domain = all( + domain.city == reference.city and domain.state_territory == reference.state_territory + for domain in domains[1:] + ) + domain = reference if use_domain else None + + requests_length = len(requests) + if requests_length == 0: + request = None + elif requests_length == 1: + request = requests[0] + else: + logger.info(f"in this loop (requests): {requests}") + reference = requests[0] + use_domain = all( + ( + (request.city == reference.city and request.state_territory == reference.state_territory) + or ( + request.suborganization_city == reference.suborganization_city + and request.suborganization_state_territory == reference.suborganization_state_territory + ) + ) + for request in requests[1:] + ) + request = reference if use_domain else None + + if not domain and not request: + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." + TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + break # PRIORITY: # 1. Domain info @@ -414,3 +466,9 @@ class Command(BaseCommand): logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) + + def locations_match(item, reference): + return (item.city == reference.city and item.state_territory == reference.state_territory) or ( + item.suborganization_city == reference.suborganization_city + and item.suborganization_state_territory == reference.suborganization_state_territory + ) From 1456104df673bf25b031c578ee2e1a6ea6c99013 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:59:19 -0700 Subject: [PATCH 53/58] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 0b2fd0887..82d2e16a4 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -407,7 +407,10 @@ class Command(BaseCommand): logger.info(f"in this loop (domains): {domains}") reference = domains[0] use_domain = all( - domain.city == reference.city and domain.state_territory == reference.state_territory + domain.city + and domain.state_territory + and domain.city == reference.city + and domain.state_territory == reference.state_territory for domain in domains[1:] ) domain = reference if use_domain else None @@ -422,9 +425,16 @@ class Command(BaseCommand): reference = requests[0] use_domain = all( ( - (request.city == reference.city and request.state_territory == reference.state_territory) + ( + request.city + and request.state_territory + and request.city == reference.city + and request.state_territory == reference.state_territory + ) or ( - request.suborganization_city == reference.suborganization_city + request.suborganization_city + and request.suborganization_state_territory + and request.suborganization_city == reference.suborganization_city and request.suborganization_state_territory == reference.suborganization_state_territory ) ) From 06623c5f8e2e4f49deca02a176fc5299b7a9dfff Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:03:44 -0700 Subject: [PATCH 54/58] Update create_federal_portfolio.py --- src/registrar/management/commands/create_federal_portfolio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 82d2e16a4..849e2cfaa 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -445,7 +445,7 @@ class Command(BaseCommand): if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - break + continue # PRIORITY: # 1. Domain info From 640fda873b40ff8baf2839a1c13061c114e3eae8 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:11:38 -0700 Subject: [PATCH 55/58] Update create_federal_portfolio.py --- .../commands/create_federal_portfolio.py | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 849e2cfaa..be0155dc5 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -398,49 +398,41 @@ class Command(BaseCommand): domains = domains_dict.get(normalized_suborg_name, []) requests = requests_dict.get(normalized_suborg_name, []) - domains_length = len(domains) - if domains_length == 0: - domain = None - elif domains_length == 1: - domain = domains[0] - else: - logger.info(f"in this loop (domains): {domains}") + # Try to get matching domain info + domain = None + if domains: reference = domains[0] - use_domain = all( - domain.city - and domain.state_territory - and domain.city == reference.city - and domain.state_territory == reference.state_territory - for domain in domains[1:] + locations_match = all( + d.city + and d.state_territory + and d.city == reference.city + and d.state_territory == reference.state_territory + for d in domains ) - domain = reference if use_domain else None + if locations_match: + domain = reference - requests_length = len(requests) - if requests_length == 0: - request = None - elif requests_length == 1: - request = requests[0] - else: - logger.info(f"in this loop (requests): {requests}") + # Try to get matching request info + request = None + if requests: reference = requests[0] - use_domain = all( + locations_match = all( ( - ( - request.city - and request.state_territory - and request.city == reference.city - and request.state_territory == reference.state_territory - ) - or ( - request.suborganization_city - and request.suborganization_state_territory - and request.suborganization_city == reference.suborganization_city - and request.suborganization_state_territory == reference.suborganization_state_territory - ) + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory ) - for request in requests[1:] + or ( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory + ) + for r in requests ) - request = reference if use_domain else None + if locations_match: + request = reference if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." From a48215faec998275f26f695d3afc30742d97a278 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:23:11 -0700 Subject: [PATCH 56/58] Account for duplicate org names --- .../commands/create_federal_portfolio.py | 128 +++++++++--------- .../tests/test_management_scripts.py | 98 ++++++++++++++ 2 files changed, 162 insertions(+), 64 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index be0155dc5..48fee06ea 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -356,43 +356,56 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) def post_process_suborganization_fields(self, agencies): - """Post-process suborganization fields by pulling data from related domains and requests. - - This function updates suborganization city and state_territory fields based on - related domain information and domain request information. + """Updates suborganization city/state fields from domain and request data. + + Priority order for data: + 1. Domain information + 2. Domain request suborganization fields + 3. Domain request standard fields """ - # Assuming that org name, portfolio, and suborg all aren't null - # we assume that we want to add suborg info. - # as long as the org name doesnt match the portfolio name (as that implies it is the portfolio). - should_add_suborgs_filter = Q( - organization_name__isnull=False, + # Common filter between domaininformation / domain request. + # Filter by only the agencies we've updated thus far. + # Then, only process records without null portfolio, org name, or suborg name. + base_filter = Q( + federal_agency__in=agencies, portfolio__isnull=False, + organization_name__isnull=False, sub_organization__isnull=False, ) & ~Q(organization_name__iexact=F("portfolio__organization_name")) + + # First: Remove null city / state_territory values on domain info / domain requests. + # We want to add city data if there is data to add to begin with! domains = DomainInformation.objects.filter( - should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False + base_filter, + Q(city__isnull=False, state_territory__isnull=False), ) requests = DomainRequest.objects.filter( - should_add_suborgs_filter, federal_agency__in=agencies, portfolio__isnull=False + base_filter, + ( + Q(city__isnull=False, state_territory__isnull=False) | + Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) + ), ) - # Since domains / requests can share an org name, lets first create lists of duplicate names. - # If only one item exists in this list, we can just pull that info. - # If more than one exists, then we can determine what value we should choose. + # Second: Group domains and requests by normalized organization name. + # This means that later down the line we have to account for "duplicate" org names. domains_dict = {} + requests_dict = {} for domain in domains: normalized_name = normalize_string(domain.organization_name) domains_dict.setdefault(normalized_name, []).append(domain) - requests_dict = {} for request in requests: normalized_name = normalize_string(request.organization_name) requests_dict.setdefault(normalized_name, []).append(request) + # Third: Get suborganizations to update suborgs_to_edit = Suborganization.objects.filter( Q(id__in=domains.values_list("sub_organization", flat=True)) | Q(id__in=requests.values_list("sub_organization", flat=True)) ) + + # Fourth: Process each suborg to add city / state territory info for suborg in suborgs_to_edit: normalized_suborg_name = normalize_string(suborg.name) domains = domains_dict.get(normalized_suborg_name, []) @@ -402,75 +415,62 @@ class Command(BaseCommand): domain = None if domains: reference = domains[0] - locations_match = all( - d.city - and d.state_territory - and d.city == reference.city + use_location_for_domain = all( + d.city == reference.city and d.state_territory == reference.state_territory for d in domains ) - if locations_match: + if use_location_for_domain: domain = reference # Try to get matching request info + # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". + # If not, take the safe route and just skip updating this particular record. request = None + use_suborg_location_for_request = True + use_location_for_request = True if requests: reference = requests[0] - locations_match = all( - ( - r.city - and r.state_territory - and r.city == reference.city - and r.state_territory == reference.state_territory - ) - or ( - r.suborganization_city - and r.suborganization_state_territory - and r.suborganization_city == reference.suborganization_city - and r.suborganization_state_territory == reference.suborganization_state_territory - ) + use_suborg_location_for_request = all( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory for r in requests ) - if locations_match: + use_location_for_request = all( + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory + for r in requests + ) + if use_suborg_location_for_request or use_location_for_request: request = reference if not domain and not request: - message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data exists." - TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." + TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, message) continue # PRIORITY: # 1. Domain info # 2. Domain request requested suborg fields # 3. Domain request normal fields - city = None - if domain and domain.city: - city = normalize_string(domain.city, lowercase=False) - elif request and request.suborganization_city: - city = normalize_string(request.suborganization_city, lowercase=False) - elif request and request.city: - city = normalize_string(request.city, lowercase=False) + if domain: + suborg.city = normalize_string(domain.city, lowercase=False) + suborg.state_territory = domain.state_territory + elif request and use_suborg_location_for_request: + suborg.city = normalize_string(request.suborganization_city, lowercase=False) + suborg.state_territory = request.suborganization_state_territory + elif request and use_location_for_request: + suborg.city = normalize_string(request.city, lowercase=False) + suborg.state_territory = request.state_territory - state_territory = None - if domain and domain.state_territory: - state_territory = domain.state_territory - elif request and request.suborganization_state_territory: - state_territory = request.suborganization_state_territory - elif request and request.state_territory: - state_territory = request.state_territory - - if city: - suborg.city = city - - if suborg: - suborg.state_territory = state_territory - - logger.info(f"{suborg}: city: {suborg.city}, state: {suborg.state_territory}") + logger.info( + f"Added city/state_territory to suborg: {suborg}. " + f"city - {suborg.city}, state - {suborg.state_territory}" + ) + # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) - - def locations_match(item, reference): - return (item.city == reference.city and item.state_territory == reference.state_territory) or ( - item.suborganization_city == reference.suborganization_city - and item.suborganization_state_territory == reference.suborganization_state_territory - ) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 8386088c8..769e9c805 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1920,6 +1920,104 @@ class TestCreateFederalPortfolio(TestCase): suborg_3 = Suborganization.objects.get(name=self.domain_info_3.organization_name) self.assertEqual(suborg_3.city, "Third City") self.assertEqual(suborg_3.state_territory, "FL") + + def test_post_process_suborganization_fields_multiple_locations(self): + """Test suborganization field updates when multiple domains/requests exist for the same org. + Tests that: + 1. Matching locations are accepted + 2. Non-matching locations are rejected + 3. Single location is always accepted + """ + # Test case 1: Multiple domains with matching locations + self.domain_info.organization_name = "matching" + self.domain_info.city = "Same City" + self.domain_info.state_territory = "NY" + self.domain_info.save() + + completed_domain_request( + name="matching" + ) + domain_info_copy = DomainInformation.objects.create( + organization_name="matching", + city="Same City", + state_territory="NY", + federal_agency=self.domain_info.federal_agency, + portfolio=self.domain_info.portfolio, + sub_organization=self.domain_info.sub_organization + ) + + # Test case 2: Multiple domains with non-matching locations + self.domain_info_2.organization_name = "non-matching" + self.domain_info_2.city = "City One" + self.domain_info_2.state_territory = "CA" + self.domain_info_2.save() + + domain_info_2_copy = DomainInformation.objects.create( + organization_name="non-matching", + city="City Two", # Different city + state_territory="CA", + federal_agency=self.domain_info_2.federal_agency, + portfolio=self.domain_info_2.portfolio, + sub_organization=self.domain_info_2.sub_organization + ) + + # Test case 3: Multiple requests with matching locations + self.domain_request.organization_name = "matching-requests" + self.domain_request.city = "Request City" + self.domain_request.state_territory = "TX" + self.domain_request.save() + + request_copy = DomainRequest.objects.create( + organization_name="matching-requests", + city="Request City", + state_territory="TX", + federal_agency=self.domain_request.federal_agency, + portfolio=self.domain_request.portfolio, + sub_organization=self.domain_request.sub_organization + ) + + # Test case 4: Multiple requests with non-matching locations + self.domain_request_2.organization_name = "non-matching-requests" + self.domain_request_2.city = "Request One" + self.domain_request_2.state_territory = "WA" + self.domain_request_2.save() + + request_2_copy = DomainRequest.objects.create( + organization_name="non-matching-requests", + city="Request Two", # Different city + state_territory="WA", + federal_agency=self.domain_request_2.federal_agency, + portfolio=self.domain_request_2.portfolio, + sub_organization=self.domain_request_2.sub_organization + ) + + # Run the portfolio creation + self.run_create_federal_portfolio( + agency_name="Test Federal Agency", + parse_requests=True, + parse_domains=True + ) + + # Verify results + # Case 1: Should use matching domain info + suborg_1 = Suborganization.objects.get(name="matching") + self.assertEqual(suborg_1.city, "Same City") + self.assertEqual(suborg_1.state_territory, "NY") + + # Case 2: Should not update due to mismatched locations + suborg_2 = Suborganization.objects.get(name="non-matching") + self.assertIsNone(suborg_2.city) + self.assertIsNone(suborg_2.state_territory) + + # Case 3: Should use matching request info + suborg_3 = Suborganization.objects.get(name="matching-requests") + self.assertEqual(suborg_3.city, "Request City") + self.assertEqual(suborg_3.state_territory, "TX") + + # Case 4: Should not update due to mismatched locations + suborg_4 = Suborganization.objects.get(name="non-matching-requests") + self.assertIsNone(suborg_4.city) + self.assertIsNone(suborg_4.state_territory) class TestPatchSuborganizations(MockDbForIndividualTests): From af7ce078bdb30993d73976511dc14d9d85182aee Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:56:04 -0700 Subject: [PATCH 57/58] Add unit tests --- .../commands/create_federal_portfolio.py | 21 +- .../tests/test_management_scripts.py | 223 +++++++++++------- 2 files changed, 152 insertions(+), 92 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 48fee06ea..534cf2c6f 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -357,17 +357,17 @@ class Command(BaseCommand): def post_process_suborganization_fields(self, agencies): """Updates suborganization city/state fields from domain and request data. - + Priority order for data: 1. Domain information - 2. Domain request suborganization fields + 2. Domain request suborganization fields 3. Domain request standard fields """ # Common filter between domaininformation / domain request. # Filter by only the agencies we've updated thus far. # Then, only process records without null portfolio, org name, or suborg name. base_filter = Q( - federal_agency__in=agencies, + federal_agency__in=agencies, portfolio__isnull=False, organization_name__isnull=False, sub_organization__isnull=False, @@ -382,9 +382,9 @@ class Command(BaseCommand): requests = DomainRequest.objects.filter( base_filter, ( - Q(city__isnull=False, state_territory__isnull=False) | - Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) - ), + Q(city__isnull=False, state_territory__isnull=False) + | Q(suborganization_city__isnull=False, suborganization_state_territory__isnull=False) + ), ) # Second: Group domains and requests by normalized organization name. @@ -416,9 +416,7 @@ class Command(BaseCommand): if domains: reference = domains[0] use_location_for_domain = all( - d.city == reference.city - and d.state_territory == reference.state_territory - for d in domains + d.city == reference.city and d.state_territory == reference.state_territory for d in domains ) if use_location_for_domain: domain = reference @@ -450,7 +448,7 @@ class Command(BaseCommand): if not domain and not request: message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." - TerminalHelper.colorful_logger(logger.info, TerminalColors.YELLOW, message) + TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) continue # PRIORITY: @@ -467,10 +465,11 @@ class Command(BaseCommand): suborg.city = normalize_string(request.city, lowercase=False) suborg.state_territory = request.state_territory - logger.info( + message = ( f"Added city/state_territory to suborg: {suborg}. " f"city - {suborg.city}, state - {suborg.state_territory}" ) + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index 769e9c805..655068493 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1845,6 +1845,7 @@ class TestCreateFederalPortfolio(TestCase): self.assertEqual(existing_portfolio.notes, "Old notes") self.assertEqual(existing_portfolio.creator, self.user) + @less_console_noise_decorator def test_post_process_suborganization_fields(self): """Test suborganization field updates from domain and request data. Also tests the priority order for updating city and state_territory: @@ -1920,104 +1921,164 @@ class TestCreateFederalPortfolio(TestCase): suborg_3 = Suborganization.objects.get(name=self.domain_info_3.organization_name) self.assertEqual(suborg_3.city, "Third City") self.assertEqual(suborg_3.state_territory, "FL") - - def test_post_process_suborganization_fields_multiple_locations(self): + + @less_console_noise_decorator + def test_post_process_suborganization_fields_duplicate_records(self): """Test suborganization field updates when multiple domains/requests exist for the same org. Tests that: - 1. Matching locations are accepted - 2. Non-matching locations are rejected - 3. Single location is always accepted + 1. City / state_territory us updated when all location info matches + 2. Updates are skipped when locations don't match + 3. Priority order is maintained across multiple records: + a. Domain information fields + b. Domain request suborganization fields + c. Domain request standard fields """ - # Test case 1: Multiple domains with matching locations - self.domain_info.organization_name = "matching" - self.domain_info.city = "Same City" - self.domain_info.state_territory = "NY" - self.domain_info.save() - - completed_domain_request( - name="matching" + # Case 1: Multiple records with all fields matching + matching_request_1 = completed_domain_request( + name="matching1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="matching org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) - domain_info_copy = DomainInformation.objects.create( - organization_name="matching", - city="Same City", - state_territory="NY", - federal_agency=self.domain_info.federal_agency, - portfolio=self.domain_info.portfolio, - sub_organization=self.domain_info.sub_organization + matching_request_1.approve() + domain_info_1 = DomainInformation.objects.get(domain_request=matching_request_1) + domain_info_1.city = "Domain Info City" + domain_info_1.state_territory = DomainRequest.StateTerritoryChoices.NEW_YORK + domain_info_1.save() + + matching_request_2 = completed_domain_request( + name="matching2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="matching org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + matching_request_2.approve() + domain_info_2 = DomainInformation.objects.get(domain_request=matching_request_2) + domain_info_2.city = "Domain Info City" + domain_info_2.state_territory = DomainRequest.StateTerritoryChoices.NEW_YORK + domain_info_2.save() - # Test case 2: Multiple domains with non-matching locations - self.domain_info_2.organization_name = "non-matching" - self.domain_info_2.city = "City One" - self.domain_info_2.state_territory = "CA" - self.domain_info_2.save() - - domain_info_2_copy = DomainInformation.objects.create( - organization_name="non-matching", - city="City Two", # Different city - state_territory="CA", - federal_agency=self.domain_info_2.federal_agency, - portfolio=self.domain_info_2.portfolio, - sub_organization=self.domain_info_2.sub_organization + # Case 2: Multiple records with only request fields (no domain info) + request_only_1 = completed_domain_request( + name="request1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="request org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + request_only_1.approve() + domain_info_3 = DomainInformation.objects.get(domain_request=request_only_1) + domain_info_3.city = None + domain_info_3.state_territory = None + domain_info_3.save() - # Test case 3: Multiple requests with matching locations - self.domain_request.organization_name = "matching-requests" - self.domain_request.city = "Request City" - self.domain_request.state_territory = "TX" - self.domain_request.save() - - request_copy = DomainRequest.objects.create( - organization_name="matching-requests", - city="Request City", - state_territory="TX", - federal_agency=self.domain_request.federal_agency, - portfolio=self.domain_request.portfolio, - sub_organization=self.domain_request.sub_organization + request_only_2 = completed_domain_request( + name="request2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="request org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + suborganization_city="Suborg City", + suborganization_state_territory=DomainRequest.StateTerritoryChoices.CALIFORNIA, + federal_agency=self.federal_agency, ) + request_only_2.approve() + domain_info_4 = DomainInformation.objects.get(domain_request=request_only_2) + domain_info_4.city = None + domain_info_4.state_territory = None + domain_info_4.save() - # Test case 4: Multiple requests with non-matching locations - self.domain_request_2.organization_name = "non-matching-requests" - self.domain_request_2.city = "Request One" - self.domain_request_2.state_territory = "WA" - self.domain_request_2.save() - - request_2_copy = DomainRequest.objects.create( - organization_name="non-matching-requests", - city="Request Two", # Different city - state_territory="WA", - federal_agency=self.domain_request_2.federal_agency, - portfolio=self.domain_request_2.portfolio, - sub_organization=self.domain_request_2.sub_organization + # Case 3: Multiple records with only standard fields (no suborg) + standard_only_1 = completed_domain_request( + name="standard1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="standard org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + federal_agency=self.federal_agency, ) + standard_only_1.approve() + domain_info_5 = DomainInformation.objects.get(domain_request=standard_only_1) + domain_info_5.city = None + domain_info_5.state_territory = None + domain_info_5.save() - # Run the portfolio creation - self.run_create_federal_portfolio( - agency_name="Test Federal Agency", - parse_requests=True, - parse_domains=True + standard_only_2 = completed_domain_request( + name="standard2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="standard org", + city="Standard City", + state_territory=DomainRequest.StateTerritoryChoices.TEXAS, + federal_agency=self.federal_agency, ) + standard_only_2.approve() + domain_info_6 = DomainInformation.objects.get(domain_request=standard_only_2) + domain_info_6.city = None + domain_info_6.state_territory = None + domain_info_6.save() - # Verify results - # Case 1: Should use matching domain info - suborg_1 = Suborganization.objects.get(name="matching") - self.assertEqual(suborg_1.city, "Same City") - self.assertEqual(suborg_1.state_territory, "NY") + # Case 4: Multiple records with mismatched locations + mismatch_request_1 = completed_domain_request( + name="mismatch1.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="mismatch org", + city="City One", + state_territory=DomainRequest.StateTerritoryChoices.FLORIDA, + federal_agency=self.federal_agency, + ) + mismatch_request_1.approve() + domain_info_5 = DomainInformation.objects.get(domain_request=mismatch_request_1) + domain_info_5.city = "Different City" + domain_info_5.state_territory = DomainRequest.StateTerritoryChoices.ALASKA + domain_info_5.save() - # Case 2: Should not update due to mismatched locations - suborg_2 = Suborganization.objects.get(name="non-matching") - self.assertIsNone(suborg_2.city) - self.assertIsNone(suborg_2.state_territory) + mismatch_request_2 = completed_domain_request( + name="mismatch2.gov", + status=DomainRequest.DomainRequestStatus.IN_REVIEW, + organization_name="mismatch org", + city="City Two", + state_territory=DomainRequest.StateTerritoryChoices.HAWAII, + federal_agency=self.federal_agency, + ) + mismatch_request_2.approve() + domain_info_6 = DomainInformation.objects.get(domain_request=mismatch_request_2) + domain_info_6.city = "Another City" + domain_info_6.state_territory = DomainRequest.StateTerritoryChoices.CALIFORNIA + domain_info_6.save() - # Case 3: Should use matching request info - suborg_3 = Suborganization.objects.get(name="matching-requests") - self.assertEqual(suborg_3.city, "Request City") - self.assertEqual(suborg_3.state_territory, "TX") + # Run the portfolio creation script + self.run_create_federal_portfolio(agency_name="Test Federal Agency", parse_requests=True, parse_domains=True) - # Case 4: Should not update due to mismatched locations - suborg_4 = Suborganization.objects.get(name="non-matching-requests") - self.assertIsNone(suborg_4.city) - self.assertIsNone(suborg_4.state_territory) + # Case 1: Should use domain info values (highest priority) + matching_suborg = Suborganization.objects.get(name="matching org") + self.assertEqual(matching_suborg.city, "Domain Info City") + self.assertEqual(matching_suborg.state_territory, DomainRequest.StateTerritoryChoices.NEW_YORK) + + # Case 2: Should use suborg values (second priority) + request_suborg = Suborganization.objects.get(name="request org") + self.assertEqual(request_suborg.city, "Suborg City") + self.assertEqual(request_suborg.state_territory, DomainRequest.StateTerritoryChoices.CALIFORNIA) + + # Case 3: Should use standard values (lowest priority) + standard_suborg = Suborganization.objects.get(name="standard org") + self.assertEqual(standard_suborg.city, "Standard City") + self.assertEqual(standard_suborg.state_territory, DomainRequest.StateTerritoryChoices.TEXAS) + + # Case 4: Should skip update due to mismatched locations + mismatch_suborg = Suborganization.objects.get(name="mismatch org") + self.assertIsNone(mismatch_suborg.city) + self.assertIsNone(mismatch_suborg.state_territory) class TestPatchSuborganizations(MockDbForIndividualTests): From 46121542437201e75b9feb2ce529459fdbe3da20 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:05:41 -0700 Subject: [PATCH 58/58] lint part 2 --- .../commands/create_federal_portfolio.py | 153 ++++++++++-------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/src/registrar/management/commands/create_federal_portfolio.py b/src/registrar/management/commands/create_federal_portfolio.py index 534cf2c6f..c56b4ff6b 100644 --- a/src/registrar/management/commands/create_federal_portfolio.py +++ b/src/registrar/management/commands/create_federal_portfolio.py @@ -106,7 +106,7 @@ class Command(BaseCommand): TerminalHelper.colorful_logger(logger.info, TerminalColors.FAIL, message) # POST PROCESS STEP: Add additional suborg info where applicable. - updated_suborg_count = self.post_process_suborganization_fields(agencies) + updated_suborg_count = self.post_process_all_suborganization_fields(agencies) message = f"Added city and state_territory information to {updated_suborg_count} suborgs." TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) @@ -355,10 +355,16 @@ class Command(BaseCommand): message = f"Added portfolio '{portfolio}' to {len(domain_infos)} domains." TerminalHelper.colorful_logger(logger.info, TerminalColors.OKGREEN, message) - def post_process_suborganization_fields(self, agencies): - """Updates suborganization city/state fields from domain and request data. + def post_process_all_suborganization_fields(self, agencies): + """Batch updates suborganization locations from domain and request data. - Priority order for data: + Args: + agencies: List of FederalAgency objects to process + + Returns: + int: Number of suborganizations updated + + Priority for location data: 1. Domain information 2. Domain request suborganization fields 3. Domain request standard fields @@ -407,69 +413,82 @@ class Command(BaseCommand): # Fourth: Process each suborg to add city / state territory info for suborg in suborgs_to_edit: - normalized_suborg_name = normalize_string(suborg.name) - domains = domains_dict.get(normalized_suborg_name, []) - requests = requests_dict.get(normalized_suborg_name, []) - - # Try to get matching domain info - domain = None - if domains: - reference = domains[0] - use_location_for_domain = all( - d.city == reference.city and d.state_territory == reference.state_territory for d in domains - ) - if use_location_for_domain: - domain = reference - - # Try to get matching request info - # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". - # If not, take the safe route and just skip updating this particular record. - request = None - use_suborg_location_for_request = True - use_location_for_request = True - if requests: - reference = requests[0] - use_suborg_location_for_request = all( - r.suborganization_city - and r.suborganization_state_territory - and r.suborganization_city == reference.suborganization_city - and r.suborganization_state_territory == reference.suborganization_state_territory - for r in requests - ) - use_location_for_request = all( - r.city - and r.state_territory - and r.city == reference.city - and r.state_territory == reference.state_territory - for r in requests - ) - if use_suborg_location_for_request or use_location_for_request: - request = reference - - if not domain and not request: - message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." - TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) - continue - - # PRIORITY: - # 1. Domain info - # 2. Domain request requested suborg fields - # 3. Domain request normal fields - if domain: - suborg.city = normalize_string(domain.city, lowercase=False) - suborg.state_territory = domain.state_territory - elif request and use_suborg_location_for_request: - suborg.city = normalize_string(request.suborganization_city, lowercase=False) - suborg.state_territory = request.suborganization_state_territory - elif request and use_location_for_request: - suborg.city = normalize_string(request.city, lowercase=False) - suborg.state_territory = request.state_territory - - message = ( - f"Added city/state_territory to suborg: {suborg}. " - f"city - {suborg.city}, state - {suborg.state_territory}" - ) - TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message) + self.post_process_suborganization_fields(suborg, domains_dict, requests_dict) # Fifth: Perform a bulk update return Suborganization.objects.bulk_update(suborgs_to_edit, ["city", "state_territory"]) + + def post_process_suborganization_fields(self, suborg, domains_dict, requests_dict): + """Updates a single suborganization's location data if valid. + + Args: + suborg: Suborganization to update + domains_dict: Dict of domain info records grouped by org name + requests_dict: Dict of domain requests grouped by org name + + Priority matches parent method. Updates are skipped if location data conflicts + between multiple records of the same type. + """ + normalized_suborg_name = normalize_string(suborg.name) + domains = domains_dict.get(normalized_suborg_name, []) + requests = requests_dict.get(normalized_suborg_name, []) + + # Try to get matching domain info + domain = None + if domains: + reference = domains[0] + use_location_for_domain = all( + d.city == reference.city and d.state_territory == reference.state_territory for d in domains + ) + if use_location_for_domain: + domain = reference + + # Try to get matching request info + # Uses consensus: if all city / state_territory info matches, then we can assume the data is "good". + # If not, take the safe route and just skip updating this particular record. + request = None + use_suborg_location_for_request = True + use_location_for_request = True + if requests: + reference = requests[0] + use_suborg_location_for_request = all( + r.suborganization_city + and r.suborganization_state_territory + and r.suborganization_city == reference.suborganization_city + and r.suborganization_state_territory == reference.suborganization_state_territory + for r in requests + ) + use_location_for_request = all( + r.city + and r.state_territory + and r.city == reference.city + and r.state_territory == reference.state_territory + for r in requests + ) + if use_suborg_location_for_request or use_location_for_request: + request = reference + + if not domain and not request: + message = f"Skipping adding city / state_territory information to suborg: {suborg}. Bad data." + TerminalHelper.colorful_logger(logger.warning, TerminalColors.YELLOW, message) + return + + # PRIORITY: + # 1. Domain info + # 2. Domain request requested suborg fields + # 3. Domain request normal fields + if domain: + suborg.city = normalize_string(domain.city, lowercase=False) + suborg.state_territory = domain.state_territory + elif request and use_suborg_location_for_request: + suborg.city = normalize_string(request.suborganization_city, lowercase=False) + suborg.state_territory = request.suborganization_state_territory + elif request and use_location_for_request: + suborg.city = normalize_string(request.city, lowercase=False) + suborg.state_territory = request.state_territory + + message = ( + f"Added city/state_territory to suborg: {suborg}. " + f"city - {suborg.city}, state - {suborg.state_territory}" + ) + TerminalHelper.colorful_logger(logger.info, TerminalColors.MAGENTA, message)