diff --git a/.github/workflows/security-check.yaml b/.github/workflows/security-check.yaml index aea700613..bf0498fff 100644 --- a/.github/workflows/security-check.yaml +++ b/.github/workflows/security-check.yaml @@ -2,17 +2,9 @@ name: Security checks on: push: - paths-ignore: - - 'docs/**' - - '**.md' - - '.gitignore' branches: - main pull_request: - paths-ignore: - - 'docs/**' - - '**.md' - - '.gitignore' branches: - main diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 642e9dc30..6332956f8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,10 +3,6 @@ name: Testing on: push: - paths-ignore: - - 'docs/**' - - '**.md' - - '.gitignore' branches: - main pull_request: diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 499c0840f..b64b5ea76 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -953,3 +953,40 @@ To create a specific portfolio: #### Step 1: Running the script ```docker-compose exec app ./manage.py patch_suborganizations``` + + +## Remove Non-whitelisted Portfolios +This script removes Portfolio entries from the database that are not part of a predefined list of allowed portfolios (`ALLOWED_PORTFOLIOS`). +It performs the following actions: +1. Prompts the user for confirmation before proceeding with deletions. +2. Updates related objects such as `DomainInformation`, `Domain`, and `DomainRequest` to set their `portfolio` field to `None` to prevent integrity errors. +3. Deletes associated objects such as `PortfolioInvitation`, `UserPortfolioPermission`, and `Suborganization`. +4. Logs a detailed summary of all cascading deletions and orphaned objects. + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-nl` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Running the script +To remove portfolios: +```./manage.py remove_unused_portfolios``` + +If you wish to enable debug mode for additional logging: +```./manage.py remove_unused_portfolios --debug``` + +### Running locally + +#### Step 1: Running the script +```docker-compose exec app ./manage.py remove_unused_portfolios``` + +To enable debug mode locally: +```docker-compose exec app ./manage.py remove_unused_portfolios --debug``` \ No newline at end of file diff --git a/src/package-lock.json b/src/package-lock.json index a769abdf0..5caff976c 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -7074,9 +7074,9 @@ } }, "node_modules/undici": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", - "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "version": "6.21.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz", + "integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==", "license": "MIT", "engines": { "node": ">=18.17" diff --git a/src/registrar/admin.py b/src/registrar/admin.py index e89147b11..8ecf36f52 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1222,9 +1222,9 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): class SeniorOfficialAdmin(ListHeaderAdmin): """Custom Senior Official Admin class.""" - search_fields = ["first_name", "last_name", "email"] + search_fields = ["first_name", "last_name", "email", "federal_agency__agency"] search_help_text = "Search by first name, last name or email." - list_display = ["first_name", "last_name", "email", "federal_agency"] + list_display = ["federal_agency", "first_name", "last_name", "email"] # this ordering effects the ordering of results # in autocomplete_fields for Senior Official @@ -1678,22 +1678,25 @@ class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin): parameter_name = "converted_generic_orgs" def lookups(self, request, model_admin): - converted_generic_orgs = set() + # Annotate the queryset to avoid Python-side iteration + queryset = ( + DomainInformation.objects.annotate( + converted_generic_org=Case( + When(portfolio__organization_type__isnull=False, then="portfolio__organization_type"), + When(portfolio__isnull=True, generic_org_type__isnull=False, then="generic_org_type"), + default=Value(""), + output_field=CharField(), + ) + ) + .values_list("converted_generic_org", flat=True) + .distinct() + ) - # Populate the set with tuples of (value, display value) - for domain_info in DomainInformation.objects.all(): - converted_generic_org = domain_info.converted_generic_org_type # Actual value - converted_generic_org_display = domain_info.converted_generic_org_type_display # Display value + # Filter out empty results and return sorted list of unique values + return sorted([(org, DomainRequest.OrganizationChoices.get_org_label(org)) for org in queryset if org]) - if converted_generic_org: - converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display - - # Sort the set by display value - return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value - - # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if a generic org is selected in the filter + if self.value(): return queryset.filter( Q(portfolio__organization_type=self.value()) | Q(portfolio__isnull=True, generic_org_type=self.value()) @@ -2031,22 +2034,25 @@ class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): parameter_name = "converted_generic_orgs" def lookups(self, request, model_admin): - converted_generic_orgs = set() + # Annotate the queryset to avoid Python-side iteration + queryset = ( + DomainRequest.objects.annotate( + converted_generic_org=Case( + When(portfolio__organization_type__isnull=False, then="portfolio__organization_type"), + When(portfolio__isnull=True, generic_org_type__isnull=False, then="generic_org_type"), + default=Value(""), + output_field=CharField(), + ) + ) + .values_list("converted_generic_org", flat=True) + .distinct() + ) - # Populate the set with tuples of (value, display value) - for domain_request in DomainRequest.objects.all(): - converted_generic_org = domain_request.converted_generic_org_type # Actual value - converted_generic_org_display = domain_request.converted_generic_org_type_display # Display value + # Filter out empty results and return sorted list of unique values + return sorted([(org, DomainRequest.OrganizationChoices.get_org_label(org)) for org in queryset if org]) - if converted_generic_org: - converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display - - # Sort the set by display value - return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value - - # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if a generic org is selected in the filter + if self.value(): return queryset.filter( Q(portfolio__organization_type=self.value()) | Q(portfolio__isnull=True, generic_org_type=self.value()) @@ -2062,24 +2068,39 @@ class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): parameter_name = "converted_federal_types" def lookups(self, request, model_admin): - converted_federal_types = set() - - # Populate the set with tuples of (value, display value) - for domain_request in DomainRequest.objects.all(): - converted_federal_type = domain_request.converted_federal_type # Actual value - converted_federal_type_display = domain_request.converted_federal_type_display # Display value - - if converted_federal_type: - converted_federal_types.add( - (converted_federal_type, converted_federal_type_display) # Value, Display + # Annotate the queryset for efficient filtering + queryset = ( + DomainRequest.objects.annotate( + converted_federal_type=Case( + When( + portfolio__isnull=False, + portfolio__federal_agency__federal_type__isnull=False, + then="portfolio__federal_agency__federal_type", + ), + When( + portfolio__isnull=True, + federal_agency__federal_type__isnull=False, + then="federal_agency__federal_type", + ), + default=Value(""), + output_field=CharField(), ) + ) + .values_list("converted_federal_type", flat=True) + .distinct() + ) - # Sort the set by display value - return sorted(converted_federal_types, key=lambda x: x[1]) # x[1] is the display value + # Filter out empty values and return sorted unique entries + return sorted( + [ + (federal_type, BranchChoices.get_branch_label(federal_type)) + for federal_type in queryset + if federal_type + ] + ) - # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if a federal type is selected in the filter + if self.value(): return queryset.filter( Q(portfolio__federal_agency__federal_type=self.value()) | Q(portfolio__isnull=True, federal_type=self.value()) @@ -3226,59 +3247,86 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): parameter_name = "converted_generic_orgs" def lookups(self, request, model_admin): - converted_generic_orgs = set() + # Annotate the queryset to avoid Python-side iteration + queryset = ( + Domain.objects.annotate( + converted_generic_org=Case( + When( + domain_info__isnull=False, + domain_info__portfolio__organization_type__isnull=False, + then="domain_info__portfolio__organization_type", + ), + When( + domain_info__isnull=False, + domain_info__portfolio__isnull=True, + domain_info__generic_org_type__isnull=False, + then="domain_info__generic_org_type", + ), + default=Value(""), + output_field=CharField(), + ) + ) + .values_list("converted_generic_org", flat=True) + .distinct() + ) - # Populate the set with tuples of (value, display value) - for domain_info in DomainInformation.objects.all(): - converted_generic_org = domain_info.converted_generic_org_type # Actual value - converted_generic_org_display = domain_info.converted_generic_org_type_display # Display value + # Filter out empty results and return sorted list of unique values + return sorted([(org, DomainRequest.OrganizationChoices.get_org_label(org)) for org in queryset if org]) - if converted_generic_org: - converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display - - # Sort the set by display value - return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value - - # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if a generic org is selected in the filter + if self.value(): return queryset.filter( Q(domain_info__portfolio__organization_type=self.value()) | Q(domain_info__portfolio__isnull=True, domain_info__generic_org_type=self.value()) ) - return queryset class FederalTypeFilter(admin.SimpleListFilter): """Custom Federal Type filter that accomodates portfolio feature. If we have a portfolio, use the portfolio's federal type. If not, use the - federal type in the Domain Information object.""" + organization in the Domain Request object.""" title = "federal type" parameter_name = "converted_federal_types" def lookups(self, request, model_admin): - converted_federal_types = set() - - # Populate the set with tuples of (value, display value) - for domain_info in DomainInformation.objects.all(): - converted_federal_type = domain_info.converted_federal_type # Actual value - converted_federal_type_display = domain_info.converted_federal_type_display # Display value - - if converted_federal_type: - converted_federal_types.add( - (converted_federal_type, converted_federal_type_display) # Value, Display + # Annotate the queryset for efficient filtering + queryset = ( + Domain.objects.annotate( + converted_federal_type=Case( + When( + domain_info__isnull=False, + domain_info__portfolio__isnull=False, + then=F("domain_info__portfolio__federal_agency__federal_type"), + ), + When( + domain_info__isnull=False, + domain_info__portfolio__isnull=True, + domain_info__federal_type__isnull=False, + then="domain_info__federal_agency__federal_type", + ), + default=Value(""), + output_field=CharField(), ) + ) + .values_list("converted_federal_type", flat=True) + .distinct() + ) - # Sort the set by display value - return sorted(converted_federal_types, key=lambda x: x[1]) # x[1] is the display value + # Filter out empty values and return sorted unique entries + return sorted( + [ + (federal_type, BranchChoices.get_branch_label(federal_type)) + for federal_type in queryset + if federal_type + ] + ) - # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if a federal type is selected in the filter + if self.value(): return queryset.filter( - Q(domain_info__portfolio__federal_agency__federal_type=self.value()) - | Q(domain_info__portfolio__isnull=True, domain_info__federal_agency__federal_type=self.value()) + Q(domain_info__portfolio__federal_type=self.value()) + | Q(domain_info__portfolio__isnull=True, domain_info__federal_type=self.value()) ) return queryset diff --git a/src/registrar/assets/src/js/getgov/table-edit-member-domains.js b/src/registrar/assets/src/js/getgov/table-edit-member-domains.js index 86aa39c37..4f0b1d610 100644 --- a/src/registrar/assets/src/js/getgov/table-edit-member-domains.js +++ b/src/registrar/assets/src/js/getgov/table-edit-member-domains.js @@ -259,7 +259,7 @@ export class EditMemberDomainsTable extends BaseTable { // Append unassigned domains section if (this.removedDomains.length) { const unassignedHeader = document.createElement('h3'); - unassignedHeader.classList.add('header--body', 'text-primary', 'margin-bottom-1'); + unassignedHeader.classList.add('margin-bottom-1'); unassignedHeader.textContent = 'Unassigned domains'; domainAssignmentSummary.appendChild(unassignedHeader); domainAssignmentSummary.appendChild(unassignedDomainsList); @@ -268,7 +268,7 @@ export class EditMemberDomainsTable extends BaseTable { // Append assigned domains section if (this.addedDomains.length) { const assignedHeader = document.createElement('h3'); - assignedHeader.classList.add('header--body', 'text-primary', 'margin-bottom-1'); + assignedHeader.classList.add('margin-bottom-1'); assignedHeader.textContent = 'Assigned domains'; domainAssignmentSummary.appendChild(assignedHeader); domainAssignmentSummary.appendChild(assignedDomainsList); @@ -276,7 +276,7 @@ export class EditMemberDomainsTable extends BaseTable { // Append total assigned domains section const totalHeader = document.createElement('h3'); - totalHeader.classList.add('header--body', 'text-primary', 'margin-bottom-1'); + totalHeader.classList.add('margin-bottom-1'); totalHeader.textContent = 'Total assigned domains'; domainAssignmentSummary.appendChild(totalHeader); const totalCount = document.createElement('p'); diff --git a/src/registrar/assets/src/js/getgov/table-members.js b/src/registrar/assets/src/js/getgov/table-members.js index 99a7fc652..2e3130a3e 100644 --- a/src/registrar/assets/src/js/getgov/table-members.js +++ b/src/registrar/assets/src/js/getgov/table-members.js @@ -245,7 +245,7 @@ export class MembersTable extends BaseTable { // Only generate HTML if the member has one or more assigned domains if (num_domains > 0) { domainsHTML += "
"; - domainsHTML += "

Domains assigned

"; + domainsHTML += "

Domains assigned

"; domainsHTML += `

This member is assigned to ${num_domains} domains:

`; domainsHTML += "