Merge branch 'main' into dk/2730-domain-overview

This commit is contained in:
David Kennedy 2024-11-26 20:07:50 -05:00
commit 84c4eec730
No known key found for this signature in database
GPG key ID: 6528A5386E66B96B
34 changed files with 916 additions and 185 deletions

50
.github/workflows/load-fixtures.yaml vendored Normal file
View file

@ -0,0 +1,50 @@
# Manually load fixtures to an environment of choice.
name: Load fixtures
run-name: Manually load fixtures to sandbox of choice
on:
workflow_dispatch:
inputs:
environment:
description: Which environment should we load data for?
type: 'choice'
options:
- ab
- backup
- el
- cb
- dk
- es
- gd
- ko
- ky
- nl
- rb
- rh
- rjm
- meoward
- bob
- hotgov
- litterbox
- ms
- ad
- ag
jobs:
load-fixtures:
runs-on: ubuntu-latest
env:
CF_USERNAME: CF_${{ github.event.inputs.environment }}_USERNAME
CF_PASSWORD: CF_${{ github.event.inputs.environment }}_PASSWORD
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
- name: Load fake data for ${{ github.event.inputs.environment }}
uses: cloud-gov/cg-cli-tools@main
with:
cf_username: ${{ secrets[env.CF_USERNAME] }}
cf_password: ${{ secrets[env.CF_PASSWORD] }}
cf_org: cisa-dotgov
cf_space: ${{ github.event.inputs.environment }}
cf_command: "run-task getgov-${{ github.event.inputs.environment }} --command 'python manage.py load' --name loaddata"

File diff suppressed because one or more lines are too long

View file

@ -2922,9 +2922,13 @@ document.addEventListener("DOMContentLoaded", () => {
const radioFieldset = document.getElementById(`id_${formPrefix}-requesting_entity_is_suborganization__fieldset`);
const radios = radioFieldset?.querySelectorAll(`input[name="${formPrefix}-requesting_entity_is_suborganization"]`);
const select = document.getElementById(`id_${formPrefix}-sub_organization`);
const selectParent = select?.parentElement;
const suborgContainer = document.getElementById("suborganization-container");
const suborgDetailsContainer = document.getElementById("suborganization-container__details");
if (!radios || !select || !suborgContainer || !suborgDetailsContainer) return;
const subOrgCreateNewOption = document.getElementById("option-to-add-suborg")?.value;
// Make sure all crucial page elements exist before proceeding.
// This more or less ensures that we are on the Requesting Entity page, and not elsewhere.
if (!radios || !select || !selectParent || !suborgContainer || !suborgDetailsContainer) return;
// requestingSuborganization: This just broadly determines if they're requesting a suborg at all
// requestingNewSuborganization: This variable determines if the user is trying to *create* a new suborganization or not.
@ -2935,12 +2939,12 @@ document.addEventListener("DOMContentLoaded", () => {
if (radio != null) requestingSuborganization = radio?.checked && radio.value === "True";
requestingSuborganization ? showElement(suborgContainer) : hideElement(suborgContainer);
requestingNewSuborganization.value = requestingSuborganization && select.value === "other" ? "True" : "False";
requestingNewSuborganization.value === "True" ? showElement(suborgDetailsContainer) : hideElement(suborgDetailsContainer);
requestingNewSuborganization.value === "True" ? showElement(suborgDetailsContainer) : hideElement(suborgDetailsContainer);
}
// Add fake "other" option to sub_organization select
if (select && !Array.from(select.options).some(option => option.value === "other")) {
select.add(new Option("Other (enter your organization manually)", "other"));
select.add(new Option(subOrgCreateNewOption, "other"));
}
if (requestingNewSuborganization.value === "True") {

File diff suppressed because one or more lines are too long

View file

@ -30,15 +30,15 @@ body {
padding-bottom: units(6) * 2 ; //Workaround because USWDS units jump from 10 to 15
}
#wrapper.wrapper--padding-top-6 {
padding-top: units(6);
}
#wrapper.dashboard {
background-color: color('primary-lightest');
padding-top: units(5)!important;
}
#wrapper.dashboard--portfolio {
padding-top: units(4)!important;
}
#wrapper.dashboard--grey-1 {
background-color: color('gray-1');
}

View file

@ -363,7 +363,6 @@ CSP_DEFAULT_SRC = ("'self'",)
CSP_STYLE_SRC = [
"'self'",
"https://www.ssa.gov/accessibility/andi/andi.css",
"https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/css/select2.min.css",
]
CSP_SCRIPT_SRC_ELEM = [
"'self'",
@ -371,7 +370,6 @@ CSP_SCRIPT_SRC_ELEM = [
"https://cdn.jsdelivr.net/npm/chart.js",
"https://www.ssa.gov",
"https://ajax.googleapis.com",
"https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/js/select2.min.js",
]
CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/", "https://www.ssa.gov/accessibility/andi/andi.js"]
CSP_INCLUDE_NONCE_IN = ["script-src-elem", "style-src"]

View file

@ -21,6 +21,7 @@ from registrar.views.report_views import (
ExportDomainRequestDataFull,
ExportDataTypeUser,
ExportDataTypeRequests,
ExportMembersPortfolio,
)
# --jsons
@ -239,6 +240,11 @@ urlpatterns = [
name="get-rejection-email-for-user-json",
),
path("admin/", admin.site.urls),
path(
"reports/export_members_portfolio/",
ExportMembersPortfolio.as_view(),
name="export_members_portfolio",
),
path(
"reports/export_data_type_user/",
ExportDataTypeUser.as_view(),

View file

@ -99,5 +99,19 @@ def portfolio_permissions(request):
def is_widescreen_mode(request):
widescreen_paths = ["/domains/", "/requests/", "/members/"]
return {"is_widescreen_mode": any(path in request.path for path in widescreen_paths) or request.path == "/"}
widescreen_paths = []
portfolio_widescreen_paths = [
"/domains/",
"/requests/",
"/request/",
"/no-organization-requests/",
"/no-organization-domains/",
"/domain-request/",
]
is_widescreen = any(path in request.path for path in widescreen_paths) or request.path == "/"
is_portfolio_widescreen = bool(
hasattr(request.user, "is_org_user")
and request.user.is_org_user(request)
and any(path in request.path for path in portfolio_widescreen_paths)
)
return {"is_widescreen_mode": is_widescreen or is_portfolio_widescreen}

View file

@ -6,8 +6,10 @@ from faker import Faker
from django.db import transaction
from registrar.fixtures.fixtures_portfolios import PortfolioFixture
from registrar.fixtures.fixtures_suborganizations import SuborganizationFixture
from registrar.fixtures.fixtures_users import UserFixture
from registrar.models import User, DomainRequest, DraftDomain, Contact, Website, FederalAgency
from registrar.models.domain import Domain
from registrar.models.portfolio import Portfolio
from registrar.models.suborganization import Suborganization
@ -101,8 +103,13 @@ class DomainRequestFixture:
}
@classmethod
def fake_dot_gov(cls):
return f"{fake.slug()}.gov"
def fake_dot_gov(cls, max_attempts=100):
"""Generate a unique .gov domain name without using an infinite loop."""
for _ in range(max_attempts):
fake_name = f"{fake.slug()}.gov"
if not Domain.objects.filter(name=fake_name).exists():
return DraftDomain.objects.create(name=fake_name)
raise RuntimeError(f"Failed to generate a unique .gov domain after {max_attempts} attempts")
@classmethod
def fake_expiration_date(cls):
@ -189,7 +196,9 @@ class DomainRequestFixture:
if not request.requested_domain:
if "requested_domain" in request_dict and request_dict["requested_domain"] is not None:
return DraftDomain.objects.get_or_create(name=request_dict["requested_domain"])[0]
return DraftDomain.objects.create(name=cls.fake_dot_gov())
# Generate a unique fake domain
return cls.fake_dot_gov()
return request.requested_domain
@classmethod
@ -213,7 +222,7 @@ class DomainRequestFixture:
if not request.sub_organization:
if "sub_organization" in request_dict and request_dict["sub_organization"] is not None:
return Suborganization.objects.get_or_create(name=request_dict["sub_organization"])[0]
return cls._get_random_sub_organization()
return cls._get_random_sub_organization(request)
return request.sub_organization
@classmethod
@ -228,10 +237,19 @@ class DomainRequestFixture:
return None
@classmethod
def _get_random_sub_organization(cls):
def _get_random_sub_organization(cls, request):
try:
suborg_options = [Suborganization.objects.first(), Suborganization.objects.last()]
return random.choice(suborg_options) # nosec
# Filter Suborganizations by the request's portfolio
portfolio_suborganizations = Suborganization.objects.filter(portfolio=request.portfolio)
# Select a suborg that's defined in the fixtures
suborganization_names = [suborg["name"] for suborg in SuborganizationFixture.SUBORGS]
# Further filter by names in suborganization_names
suborganization_options = portfolio_suborganizations.filter(name__in=suborganization_names)
# Randomly choose one if any exist
return random.choice(suborganization_options) if suborganization_options.exists() else None # nosec
except Exception as e:
logger.warning(f"Expected fixture sub_organization, did not find it: {e}")
return None
@ -273,6 +291,9 @@ class DomainRequestFixture:
# Lumped under .atomic to ensure we don't make redundant DB calls.
# This bundles them all together, and then saves it in a single call.
# The atomic block will cause the code to stop executing if one instance in the
# nested iteration fails, which will cause an early exit and make it hard to debug.
# Comment out with transaction.atomic() when debugging.
with transaction.atomic():
try:
# Get the usernames of users created in the UserFixture

View file

@ -267,54 +267,24 @@ class UserFixture:
"""Loads the users into the database and assigns them to the specified group."""
logger.info(f"Going to load {len(users)} users for group {group_name}")
# Step 1: Fetch the group
group = UserGroup.objects.get(name=group_name)
# Prepare sets of existing usernames and IDs in one query
user_identifiers = [(user.get("username"), user.get("id")) for user in users]
existing_users = User.objects.filter(
username__in=[user[0] for user in user_identifiers] + [user[1] for user in user_identifiers]
).values_list("username", "id")
# Step 2: Identify new and existing users
existing_usernames, existing_user_ids = cls._get_existing_users(users)
new_users = cls._prepare_new_users(users, existing_usernames, existing_user_ids, are_superusers)
existing_usernames = set(user[0] for user in existing_users)
existing_user_ids = set(user[1] for user in existing_users)
# Filter out users with existing IDs or usernames
new_users = [
User(
id=user_data.get("id"),
first_name=user_data.get("first_name"),
last_name=user_data.get("last_name"),
username=user_data.get("username"),
email=user_data.get("email", ""),
title=user_data.get("title", "Peon"),
phone=user_data.get("phone", "2022222222"),
is_active=user_data.get("is_active", True),
is_staff=True,
is_superuser=are_superusers,
)
for user_data in users
if user_data.get("username") not in existing_usernames and user_data.get("id") not in existing_user_ids
]
# Perform bulk creation for new users
if new_users:
try:
User.objects.bulk_create(new_users)
logger.info(f"Created {len(new_users)} new users.")
except Exception as e:
logger.error(f"Unexpected error during user bulk creation: {e}")
else:
logger.info("No new users to create.")
# Step 3: Create new users
cls._create_new_users(new_users)
# Step 4: Update existing users
# Get all users to be updated (both new and existing)
created_or_existing_users = User.objects.filter(username__in=[user.get("username") for user in users])
users_to_update = cls._get_users_to_update(created_or_existing_users)
cls._update_existing_users(users_to_update)
# Filter out users who are already in the group
users_not_in_group = created_or_existing_users.exclude(groups__id=group.id)
# Add only users who are not already in the group
if users_not_in_group.exists():
group.user_set.add(*users_not_in_group)
# Step 5: Assign users to the group
cls._assign_users_to_group(group, created_or_existing_users)
logger.info(f"Users loaded for group {group_name}.")
@ -346,6 +316,76 @@ class UserFixture:
else:
logger.info("No allowed emails to load")
@staticmethod
def _get_existing_users(users):
user_identifiers = [(user.get("username"), user.get("id")) for user in users]
existing_users = User.objects.filter(
username__in=[user[0] for user in user_identifiers] + [user[1] for user in user_identifiers]
).values_list("username", "id")
existing_usernames = set(user[0] for user in existing_users)
existing_user_ids = set(user[1] for user in existing_users)
return existing_usernames, existing_user_ids
@staticmethod
def _prepare_new_users(users, existing_usernames, existing_user_ids, are_superusers):
return [
User(
id=user_data.get("id"),
first_name=user_data.get("first_name"),
last_name=user_data.get("last_name"),
username=user_data.get("username"),
email=user_data.get("email", ""),
title=user_data.get("title", "Peon"),
phone=user_data.get("phone", "2022222222"),
is_active=user_data.get("is_active", True),
is_staff=True,
is_superuser=are_superusers,
)
for user_data in users
if user_data.get("username") not in existing_usernames and user_data.get("id") not in existing_user_ids
]
@staticmethod
def _create_new_users(new_users):
if new_users:
try:
User.objects.bulk_create(new_users)
logger.info(f"Created {len(new_users)} new users.")
except Exception as e:
logger.error(f"Unexpected error during user bulk creation: {e}")
else:
logger.info("No new users to create.")
@staticmethod
def _get_users_to_update(users):
users_to_update = []
for user in users:
updated = False
if not user.title:
user.title = "Peon"
updated = True
if not user.phone:
user.phone = "2022222222"
updated = True
if not user.is_staff:
user.is_staff = True
updated = True
if updated:
users_to_update.append(user)
return users_to_update
@staticmethod
def _update_existing_users(users_to_update):
if users_to_update:
User.objects.bulk_update(users_to_update, ["is_staff", "title", "phone"])
logger.info(f"Updated {len(users_to_update)} existing users.")
@staticmethod
def _assign_users_to_group(group, users):
users_not_in_group = users.exclude(groups__id=group.id)
if users_not_in_group.exists():
group.user_set.add(*users_not_in_group)
@classmethod
def load(cls):
with transaction.atomic():

View file

@ -115,11 +115,14 @@ class RequestingEntityForm(RegistrarForm):
if is_requesting_new_suborganization:
# Validate custom suborganization fields
if not cleaned_data.get("requested_suborganization"):
self.add_error("requested_suborganization", "Requested suborganization is required.")
self.add_error("requested_suborganization", "Enter the name of your suborganization.")
if not cleaned_data.get("suborganization_city"):
self.add_error("suborganization_city", "City is required.")
self.add_error("suborganization_city", "Enter the city where your suborganization is located.")
if not cleaned_data.get("suborganization_state_territory"):
self.add_error("suborganization_state_territory", "State, territory, or military post is required.")
self.add_error(
"suborganization_state_territory",
"Select the state, territory, or military post where your suborganization is located.",
)
elif not suborganization:
self.add_error("sub_organization", "Suborganization is required.")

View file

@ -2,7 +2,12 @@ from django.db import models
from django.forms import ValidationError
from registrar.models.user_domain_role import UserDomainRole
from registrar.utility.waffle import flag_is_active_for_user
from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices
from registrar.models.utility.portfolio_helper import (
UserPortfolioPermissionChoices,
UserPortfolioRoleChoices,
DomainRequestPermissionDisplay,
MemberPermissionDisplay,
)
from .utility.time_stamped_model import TimeStampedModel
from django.contrib.postgres.fields import ArrayField
@ -106,6 +111,37 @@ class UserPortfolioPermission(TimeStampedModel):
portfolio_permissions.update(additional_permissions)
return list(portfolio_permissions)
@classmethod
def get_domain_request_permission_display(cls, roles, additional_permissions):
"""Class method to return a readable string for domain request permissions"""
# Tracks if they can view, create requests, or not do anything
all_permissions = UserPortfolioPermission.get_portfolio_permissions(roles, additional_permissions)
all_domain_perms = [
UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS,
UserPortfolioPermissionChoices.EDIT_REQUESTS,
]
if all(perm in all_permissions for perm in all_domain_perms):
return DomainRequestPermissionDisplay.VIEWER_REQUESTER
elif UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS in all_permissions:
return DomainRequestPermissionDisplay.VIEWER
else:
return DomainRequestPermissionDisplay.NONE
@classmethod
def get_member_permission_display(cls, roles, additional_permissions):
"""Class method to return a readable string for member permissions"""
# Tracks if they can view, create requests, or not do anything.
# This is different than get_domain_request_permission_display because member tracks
# permissions slightly differently.
all_permissions = UserPortfolioPermission.get_portfolio_permissions(roles, additional_permissions)
if UserPortfolioPermissionChoices.EDIT_MEMBERS in all_permissions:
return MemberPermissionDisplay.MANAGER
elif UserPortfolioPermissionChoices.VIEW_MEMBERS in all_permissions:
return MemberPermissionDisplay.VIEWER
else:
return MemberPermissionDisplay.NONE
def clean(self):
"""Extends clean method to perform additional validation, which can raise errors in django admin."""
super().clean()

View file

@ -0,0 +1,8 @@
from django.db.models.expressions import Func
class ArrayRemoveNull(Func):
"""Custom Func to use array_remove to remove null values"""
function = "array_remove"
template = "%(function)s(%(expressions)s, NULL)"

View file

@ -1,3 +1,4 @@
from registrar.utility import StrEnum
from django.db import models
@ -40,3 +41,31 @@ class UserPortfolioPermissionChoices(models.TextChoices):
@classmethod
def to_dict(cls):
return {key: value.value for key, value in cls.__members__.items()}
class DomainRequestPermissionDisplay(StrEnum):
"""Stores display values for domain request permission combinations.
Overview of values:
- VIEWER_REQUESTER: "Viewer Requester"
- VIEWER: "Viewer"
- NONE: "None"
"""
VIEWER_REQUESTER = "Viewer Requester"
VIEWER = "Viewer"
NONE = "None"
class MemberPermissionDisplay(StrEnum):
"""Stores display values for member permission combinations.
Overview of values:
- MANAGER: "Manager"
- VIEWER: "Viewer"
- NONE: "None"
"""
MANAGER = "Manager"
VIEWER = "Viewer"
NONE = "None"

View file

@ -16,10 +16,10 @@
<script src="{% static 'admin/js/vendor/jquery/jquery.min.js' %}"></script>
<!-- Include Select2 JavaScript. Since this view technically falls outside of admin, this is needed. -->
<script src="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/js/select2.min.js"></script>
<script src="{% static 'js/select2.min.js' %}"></script>
<script type="application/javascript" src="{% static 'js/get-gov-admin-extra.js' %}" defer></script>
<link href="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/css/select2.min.css" rel="stylesheet" />
<link href="{% static 'css/select2.min.css' %}" rel="stylesheet" />
{% endblock %}
{% block breadcrumbs %}

View file

@ -158,7 +158,9 @@
{% endblock header %}
{% block wrapper %}
{% block wrapperdiv %}
<div id="wrapper">
{% endblock wrapperdiv %}
{% block messages %}
{% if messages %}
<ul class="messages">

View file

@ -4,6 +4,12 @@
{% block title %}Thanks for your domain request! | {% endblock %}
{% comment %} Same as the old wrapper implementation but with padding-top-4 {% endcomment %}
{% block wrapperdiv %}
<div id="wrapper" class="wrapper--padding-top-6">
{% endblock wrapperdiv %}
{% block content %}
<main id="main-content" class="grid-container register-form-step">
<span class="display-flex flex-align-center" >
@ -28,8 +34,8 @@
<li>Your requested domain meets our naming requirements.</li>
</ul>
<p> Well email you if we have questions. Well also email you as soon as we complete our review. You can <a href="{% url 'home' %}">check the status</a>
of your request at any time on the registrar homepage.</p>
<p> Well email you if we have questions. Well also email you as soon as we complete our review. You can <a href="{% if portfolio %}{% url 'domain-requests' %}{% else %}{% url 'home' %}{% endif %}">check the status</a>
of your request at any time on the registrar.</p>
<p> <a class="usa-link" rel="noopener noreferrer" target="_blank" href="{% public_site_url 'contact' %}">Contact us if you need help during this process</a>.</p>

View file

@ -2,11 +2,16 @@
{% load field_helpers url_helpers %}
{% block form_instructions %}
<p>To help with our review, we need to understand whether the domain you're requesting will be used by the Department of Energy or by one of its suborganizations.</p>
<p>To help with our review, we need to understand whether the domain you're requesting will be used by {{ portfolio }} or by one of its suborganizations.</p>
<p>We define a suborganization as any entity (agency, bureau, office) that falls under the overarching organization.</p>
{% endblock %}
{% block form_fields %}
{% comment %}
Store the other option in a variable to be used by the js function handleRequestingEntity.
Injected into the 'sub_organization' option list.
{% endcomment %}
<input id="option-to-add-suborg" class="display-none" value="Other (enter your suborganization manually)"/>
<fieldset class="usa-fieldset">
<legend>
<h2>Who will use the domain youre requesting?</h2>
@ -33,8 +38,8 @@
<div id="suborganization-container" class="margin-top-4">
<h2>Add suborganization information</h2>
<p>
This information will be published in <a class="usa-link usa-link--always-blue" href="{% public_site_url 'about/data' %}">.govs public data</a>. If you dont see your suborganization in the list,
select “other” and enter the name or your suborganization.
This information will be published in <a class="usa-link usa-link--always-blue" target="_blank" href="{% public_site_url 'about/data' %}">.govs public data</a>. If you dont see your suborganization in the list,
select “other.
</p>
{% with attr_required=True %}
{% input_with_errors forms.1.sub_organization %}
@ -43,7 +48,7 @@
{% comment %} This will be toggled if a special value, "other", is selected.
Otherwise this field is invisible.
{% endcomment %}
<div id="suborganization-container__details">
<div id="suborganization-container__details" class="padding-top-2 margin-top-0">
{% with attr_required=True %}
{% input_with_errors forms.1.requested_suborganization %}
{% endwith %}

View file

@ -3,6 +3,10 @@
{% block title %}Withdraw request for {{ DomainRequest.requested_domain.name }} | {% endblock %}
{% load static url_helpers %}
{% block wrapperdiv %}
<div id="wrapper" class="wrapper--padding-top-6">
{% endblock wrapperdiv %}
{% block content %}
<div class="grid-container">
<div class="grid-col desktop:grid-offset-2 desktop:grid-col-8">

View file

@ -34,7 +34,6 @@
</ul>
</div>
<ul class="usa-nav__primary usa-accordion">
{% if not hide_domains %}
<li class="usa-nav__primary-item">
{% if has_any_domains_portfolio_permission %}
{% url 'domains' as url %}
@ -45,14 +44,13 @@
Domains
</a>
</li>
{% endif %}
<!-- <li class="usa-nav__primary-item">
<a href="#" class="usa-nav-link">
Domain groups
</a>
</li> -->
{% if has_organization_requests_flag and not hide_requests %}
{% if has_organization_requests_flag %}
<li class="usa-nav__primary-item">
<!-- user has one of the view permissions plus the edit permission, show the dropdown -->
{% if has_edit_request_portfolio_permission %}
@ -93,7 +91,7 @@
</li>
{% endif %}
{% if has_organization_members_flag and not hide_members %}
{% if has_organization_members_flag %}
<li class="usa-nav__primary-item">
<a href="{% url 'members' %}" class="usa-nav-link {% if path|is_members_subpage %} usa-current{% endif %}">
Members

View file

@ -8,7 +8,7 @@
<section class="section-outlined members margin-top-0 section-outlined--border-base-light" id="members">
<div class="section-outlined__header margin-bottom-3 grid-row">
<!-- ---------- SEARCH ---------- -->
<div class="section-outlined__search mobile:grid-col-12 desktop:grid-col-6">
<div class="section-outlined__search mobile:grid-col-12 desktop:grid-col-6 {% if is_widescreen_mode %} section-outlined__search--widescreen {% endif %}">
<section aria-label="Members search component" class="margin-top-2">
<form class="usa-search usa-search--small" method="POST" role="search">
{% csrf_token %}
@ -36,6 +36,15 @@
</form>
</section>
</div>
<div class="section-outlined__utility-button mobile-lg:padding-right-105 {% if portfolio %} mobile:grid-col-12 desktop:grid-col-6 desktop:padding-left-3{% endif %}">
<section aria-label="Domains report component" class="margin-top-205">
<a href="{% url 'export_members_portfolio' %}" class="usa-button usa-button--unstyled usa-button--with-icon usa-button--justify-right" role="button">
<svg class="usa-icon usa-icon--big" aria-hidden="true" focusable="false" role="img" width="24" height="24">
<use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
</svg>Export as CSV
</a>
</section>
</div>
</div>
<!-- ---------- MAIN TABLE ---------- -->

View file

@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block wrapper %}
<div id="wrapper" class="{% block wrapper_class %}dashboard--portfolio{% endblock %}">
<div id="wrapper" class="{% block wrapper_class %}wrapper--padding-top-6{% endblock %}">
{% block content %}
<main class="grid-container {% if is_widescreen_mode %} grid-container--widescreen {% endif %}">

View file

@ -18,7 +18,7 @@
<h2 id="domains-header" class="display-inline-block">You arent managing any domains.</h2>
{% if portfolio_administrators %}
<p>If you believe you should have access to a domain, reach out to your organizations administrators.</p>
<p>Your organizations administrators:</p>
<p>Your organization's administrators:</p>
<ul class="margin-top-0">
{% for administrator in portfolio_administrators %}
{% if administrator.email %}

View file

@ -5,13 +5,13 @@
{% block title %} Domain Requests | {% endblock %}
{% block portfolio_content %}
<h1 id="domains-header">Current domain requests</h1>
<h1 id="domains-header">Domain requests</h1>
<section class="section-outlined">
<div class="section-outlined__header margin-bottom-3">
<h2 id="domains-header" class="display-inline-block">You dont have access to domain requests.</h2>
{% if portfolio_administrators %}
<p>If you believe you should have access to a request, reach out to your organizations administrators.</p>
<p>Your organizations administrators:</p>
<p>If you believe you should have access to requests, reach out to your organizations administrators.</p>
<p>Your organization's administrators:</p>
<ul class="margin-top-0">
{% for administrator in portfolio_administrators %}
{% if administrator.email %}

View file

@ -14,12 +14,12 @@
{% endblock %}
<div id="main-content">
<h1 id="domain-requests-header">Domain requests</h1>
<h1 id="domain-requests-header" class="margin-bottom-1">Domain requests</h1>
<div class="grid-row grid-gap">
{% if has_edit_request_portfolio_permission %}
<div class="mobile:grid-col-12 tablet:grid-col-6">
<p class="margin-y-0">Domain requests can only be modified by the person who created the request.</p>
<p class="margin-y-0 maxw-mobile">Domain requests can only be modified by the person who created the request.</p>
</div>
<div class="mobile:grid-col-12 tablet:grid-col-6">

View file

@ -1,6 +1,5 @@
import os
import logging
from contextlib import contextmanager
import random
from string import ascii_uppercase
@ -29,6 +28,7 @@ from registrar.models import (
FederalAgency,
UserPortfolioPermission,
Portfolio,
PortfolioInvitation,
)
from epplibwrapper import (
commands,
@ -39,6 +39,7 @@ from epplibwrapper import (
ErrorCode,
responses,
)
from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices
from registrar.models.user_domain_role import UserDomainRole
from registrar.models.utility.contact_error import ContactError, ContactErrorCodes
@ -196,6 +197,7 @@ class GenericTestHelper(TestCase):
self.assertEqual(expected_sort_order, returned_sort_order)
@classmethod
def _mock_user_request_for_factory(self, request):
"""Adds sessionmiddleware when using factory to associate session information"""
middleware = SessionMiddleware(lambda req: req)
@ -531,6 +533,8 @@ class MockDb(TestCase):
@classmethod
@less_console_noise_decorator
def sharedSetUp(cls):
cls.mock_client_class = MagicMock()
cls.mock_client = cls.mock_client_class.return_value
username = "test_user"
first_name = "First"
last_name = "Last"
@ -540,6 +544,29 @@ class MockDb(TestCase):
cls.user = get_user_model().objects.create(
username=username, first_name=first_name, last_name=last_name, email=email, title=title, phone=phone
)
cls.meoward_user = get_user_model().objects.create(
username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com"
)
cls.lebowski_user = get_user_model().objects.create(
username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co"
)
cls.tired_user = get_user_model().objects.create(
username="ministry_of_bedtime", first_name="tired", last_name="sleepy", email="tired_sleepy@igorville.gov"
)
# Custom superuser and staff so that these do not conflict with what may be defined on what implements this.
cls.custom_superuser = create_superuser(
username="cold_superuser", first_name="cold", last_name="icy", email="icy_superuser@igorville.gov"
)
cls.custom_staffuser = create_user(
username="warm_staff", first_name="warm", last_name="cozy", email="cozy_staffuser@igorville.gov"
)
cls.federal_agency_1, _ = FederalAgency.objects.get_or_create(agency="World War I Centennial Commission")
cls.federal_agency_2, _ = FederalAgency.objects.get_or_create(agency="Armed Forces Retirement Home")
cls.portfolio_1, _ = Portfolio.objects.get_or_create(
creator=cls.custom_superuser, federal_agency=cls.federal_agency_1
)
current_date = get_time_aware_date(datetime(2024, 4, 2))
# Create start and end dates using timedelta
@ -547,9 +574,6 @@ class MockDb(TestCase):
cls.end_date = current_date + timedelta(days=2)
cls.start_date = current_date - timedelta(days=2)
cls.federal_agency_1, _ = FederalAgency.objects.get_or_create(agency="World War I Centennial Commission")
cls.federal_agency_2, _ = FederalAgency.objects.get_or_create(agency="Armed Forces Retirement Home")
cls.domain_1, _ = Domain.objects.get_or_create(
name="cdomain1.gov", state=Domain.State.READY, first_ready=get_time_aware_date(datetime(2024, 4, 2))
)
@ -596,9 +620,14 @@ class MockDb(TestCase):
federal_agency=cls.federal_agency_1,
federal_type="executive",
is_election_board=False,
portfolio=cls.portfolio_1,
)
cls.domain_information_2, _ = DomainInformation.objects.get_or_create(
creator=cls.user, domain=cls.domain_2, generic_org_type="interstate", is_election_board=True
creator=cls.user,
domain=cls.domain_2,
generic_org_type="interstate",
is_election_board=True,
portfolio=cls.portfolio_1,
)
cls.domain_information_3, _ = DomainInformation.objects.get_or_create(
creator=cls.user,
@ -671,14 +700,6 @@ class MockDb(TestCase):
is_election_board=False,
)
cls.meoward_user = get_user_model().objects.create(
username="meoward_username", first_name="first_meoward", last_name="last_meoward", email="meoward@rocks.com"
)
cls.lebowski_user = get_user_model().objects.create(
username="big_lebowski", first_name="big", last_name="lebowski", email="big_lebowski@dude.co"
)
_, created = UserDomainRole.objects.get_or_create(
user=cls.meoward_user, domain=cls.domain_1, role=UserDomainRole.Roles.MANAGER
)
@ -709,6 +730,12 @@ class MockDb(TestCase):
status=DomainInvitation.DomainInvitationStatus.RETRIEVED,
)
_, created = DomainInvitation.objects.get_or_create(
email=cls.meoward_user.email,
domain=cls.domain_11,
status=DomainInvitation.DomainInvitationStatus.RETRIEVED,
)
_, created = DomainInvitation.objects.get_or_create(
email="woofwardthethird@rocks.com",
domain=cls.domain_1,
@ -723,6 +750,85 @@ class MockDb(TestCase):
email="squeaker@rocks.com", domain=cls.domain_10, status=DomainInvitation.DomainInvitationStatus.INVITED
)
cls.portfolio_invitation_1, _ = PortfolioInvitation.objects.get_or_create(
email=cls.meoward_user.email,
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.EDIT_MEMBERS],
)
cls.portfolio_invitation_2, _ = PortfolioInvitation.objects.get_or_create(
email=cls.lebowski_user.email,
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.VIEW_MEMBERS],
)
cls.portfolio_invitation_3, _ = PortfolioInvitation.objects.get_or_create(
email=cls.tired_user.email,
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS],
)
cls.portfolio_invitation_4, _ = PortfolioInvitation.objects.get_or_create(
email=cls.custom_superuser.email,
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN],
additional_permissions=[
UserPortfolioPermissionChoices.VIEW_MEMBERS,
UserPortfolioPermissionChoices.EDIT_MEMBERS,
UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS,
UserPortfolioPermissionChoices.EDIT_REQUESTS,
],
)
cls.portfolio_invitation_5, _ = PortfolioInvitation.objects.get_or_create(
email=cls.custom_staffuser.email,
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN],
)
# Add some invitations that we never retireve
PortfolioInvitation.objects.get_or_create(
email="nonexistentmember_1@igorville.gov",
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.EDIT_MEMBERS],
)
PortfolioInvitation.objects.get_or_create(
email="nonexistentmember_2@igorville.gov",
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.VIEW_MEMBERS],
)
PortfolioInvitation.objects.get_or_create(
email="nonexistentmember_3@igorville.gov",
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_MEMBER],
additional_permissions=[UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS],
)
PortfolioInvitation.objects.get_or_create(
email="nonexistentmember_4@igorville.gov",
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN],
additional_permissions=[
UserPortfolioPermissionChoices.VIEW_MEMBERS,
UserPortfolioPermissionChoices.EDIT_MEMBERS,
UserPortfolioPermissionChoices.VIEW_ALL_REQUESTS,
UserPortfolioPermissionChoices.EDIT_REQUESTS,
],
)
PortfolioInvitation.objects.get_or_create(
email="nonexistentmember_5@igorville.gov",
portfolio=cls.portfolio_1,
roles=[UserPortfolioRoleChoices.ORGANIZATION_ADMIN],
)
with less_console_noise():
cls.domain_request_1 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED,
@ -731,10 +837,12 @@ class MockDb(TestCase):
cls.domain_request_2 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.IN_REVIEW,
name="city2.gov",
portfolio=cls.portfolio_1,
)
cls.domain_request_3 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED,
name="city3.gov",
portfolio=cls.portfolio_1,
)
cls.domain_request_4 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED,
@ -749,6 +857,7 @@ class MockDb(TestCase):
cls.domain_request_6 = completed_domain_request(
status=DomainRequest.DomainRequestStatus.STARTED,
name="city6.gov",
portfolio=cls.portfolio_1,
)
cls.domain_request_3.submit()
cls.domain_request_4.submit()
@ -797,6 +906,7 @@ class MockDb(TestCase):
UserPortfolioPermission.objects.all().delete()
User.objects.all().delete()
DomainInvitation.objects.all().delete()
PortfolioInvitation.objects.all().delete()
cls.federal_agency_1.delete()
cls.federal_agency_2.delete()
@ -837,17 +947,18 @@ def mock_user():
return mock_user
def create_superuser():
def create_superuser(**kwargs):
"""Creates a analyst user with is_staff=True and the group full_access_group"""
User = get_user_model()
p = "adminpass"
user = User.objects.create_user(
username="superuser",
email="admin@example.com",
first_name="first",
last_name="last",
is_staff=True,
password=p,
phone="8003111234",
username=kwargs.get("username", "superuser"),
email=kwargs.get("email", "admin@example.com"),
first_name=kwargs.get("first_name", "first"),
last_name=kwargs.get("last_name", "last"),
is_staff=kwargs.get("is_staff", True),
password=kwargs.get("password", p),
phone=kwargs.get("phone", "8003111234"),
)
# Retrieve the group or create it if it doesn't exist
group, _ = UserGroup.objects.get_or_create(name="full_access_group")
@ -856,18 +967,19 @@ def create_superuser():
return user
def create_user():
def create_user(**kwargs):
"""Creates a analyst user with is_staff=True and the group cisa_analysts_group"""
User = get_user_model()
p = "userpass"
user = User.objects.create_user(
username="staffuser",
email="staff@example.com",
first_name="first",
last_name="last",
is_staff=True,
title="title",
password=p,
phone="8003111234",
username=kwargs.get("username", "staffuser"),
email=kwargs.get("email", "staff@example.com"),
first_name=kwargs.get("first_name", "first"),
last_name=kwargs.get("last_name", "last"),
is_staff=kwargs.get("is_staff", True),
title=kwargs.get("title", "title"),
password=kwargs.get("password", p),
phone=kwargs.get("phone", "8003111234"),
)
# Retrieve the group or create it if it doesn't exist
group, _ = UserGroup.objects.get_or_create(name="cisa_analysts_group")

View file

@ -5,6 +5,8 @@ from registrar.models import (
DomainRequest,
Domain,
UserDomainRole,
PortfolioInvitation,
User,
)
from registrar.models import Portfolio, DraftDomain
from registrar.models.user_portfolio_permission import UserPortfolioPermission
@ -22,6 +24,7 @@ from registrar.utility.csv_export import (
DomainRequestExport,
DomainRequestGrowth,
DomainRequestDataFull,
MemberExport,
get_default_start_date,
get_default_end_date,
)
@ -42,9 +45,14 @@ from .common import (
get_wsgi_request_object,
less_console_noise,
get_time_aware_date,
GenericTestHelper,
)
from waffle.testutils import override_flag
from datetime import datetime
from django.contrib.admin.models import LogEntry, ADDITION
from django.contrib.contenttypes.models import ContentType
class CsvReportsTest(MockDbForSharedTests):
"""Tests to determine if we are uploading our reports correctly."""
@ -794,6 +802,104 @@ class ExportDataTest(MockDbForIndividualTests, MockEppLib):
self.assertEqual(csv_content, expected_content)
class MemberExportTest(MockDbForIndividualTests, MockEppLib):
def setUp(self):
"""Override of the base setUp to add a request factory"""
super().setUp()
self.factory = RequestFactory()
@override_flag("organization_feature", active=True)
@override_flag("organization_members", active=True)
@less_console_noise_decorator
def test_member_export(self):
"""Tests the member export report by comparing the csv output."""
# == Data setup == #
# Set last_login for some users
active_date = timezone.make_aware(datetime(2024, 2, 1))
User.objects.filter(id__in=[self.custom_superuser.id, self.custom_staffuser.id]).update(last_login=active_date)
# Create a logentry for meoward, created by lebowski to test invited_by.
content_type = ContentType.objects.get_for_model(PortfolioInvitation)
LogEntry.objects.create(
user=self.lebowski_user,
content_type=content_type,
object_id=self.portfolio_invitation_1.id,
object_repr=str(self.portfolio_invitation_1),
action_flag=ADDITION,
change_message="Created invitation",
action_time=timezone.make_aware(datetime(2023, 4, 12)),
)
# Create log entries for each remaining invitation. Exclude meoward and tired_user.
for invitation in PortfolioInvitation.objects.exclude(
id__in=[self.portfolio_invitation_1.id, self.portfolio_invitation_3.id]
):
LogEntry.objects.create(
user=self.custom_staffuser,
content_type=content_type,
object_id=invitation.id,
object_repr=str(invitation),
action_flag=ADDITION,
change_message="Created invitation",
action_time=timezone.make_aware(datetime(2024, 1, 15)),
)
# Retrieve invitations
with boto3_mocking.clients.handler_for("sesv2", self.mock_client_class):
self.meoward_user.check_portfolio_invitations_on_login()
self.lebowski_user.check_portfolio_invitations_on_login()
self.tired_user.check_portfolio_invitations_on_login()
self.custom_superuser.check_portfolio_invitations_on_login()
self.custom_staffuser.check_portfolio_invitations_on_login()
# Update the created at date on UserPortfolioPermission, so we can test a consistent date.
UserPortfolioPermission.objects.filter(portfolio=self.portfolio_1).update(
created_at=timezone.make_aware(datetime(2022, 4, 1))
)
# == End of data setup == #
# Create a request and add the user to the request
request = self.factory.get("/")
request.user = self.user
self.maxDiff = None
# Add portfolio to session
request = GenericTestHelper._mock_user_request_for_factory(request)
request.session["portfolio"] = self.portfolio_1
# Create a CSV file in memory
csv_file = StringIO()
# Call the export function
MemberExport.export_data_to_csv(csv_file, request=request)
# Reset the CSV file's position to the beginning
csv_file.seek(0)
# Read the content into a variable
csv_content = csv_file.read()
expected_content = (
# Header
"Email,Organization admin,Invited by,Joined date,Last active,Domain requests,"
"Member management,Domain management,Number of domains,Domains\n"
# Content
"meoward@rocks.com,False,big_lebowski@dude.co,2022-04-01,Invalid date,None,"
'Manager,True,2,"adomain2.gov,cdomain1.gov"\n'
"big_lebowski@dude.co,False,help@get.gov,2022-04-01,Invalid date,None,Viewer,True,1,cdomain1.gov\n"
"tired_sleepy@igorville.gov,False,System,2022-04-01,Invalid date,Viewer,None,False,0,\n"
"icy_superuser@igorville.gov,True,help@get.gov,2022-04-01,2024-02-01,Viewer Requester,Manager,False,0,\n"
"cozy_staffuser@igorville.gov,True,help@get.gov,2022-04-01,2024-02-01,Viewer Requester,None,False,0,\n"
"nonexistentmember_1@igorville.gov,False,help@get.gov,Unretrieved,Invited,None,Manager,False,0,\n"
"nonexistentmember_2@igorville.gov,False,help@get.gov,Unretrieved,Invited,None,Viewer,False,0,\n"
"nonexistentmember_3@igorville.gov,False,help@get.gov,Unretrieved,Invited,Viewer,None,False,0,\n"
"nonexistentmember_4@igorville.gov,True,help@get.gov,Unretrieved,"
"Invited,Viewer Requester,Manager,False,0,\n"
"nonexistentmember_5@igorville.gov,True,help@get.gov,Unretrieved,Invited,Viewer Requester,None,False,0,\n"
)
# Normalize line endings and remove commas,
# spaces and leading/trailing whitespace
csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
self.assertEqual(csv_content, expected_content)
class HelperFunctions(MockDbForSharedTests):
"""This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy."""

View file

@ -2162,7 +2162,7 @@ class TestRequestingEntity(WebTest):
self.assertContains(response, "Add suborganization information")
# We expect to see the portfolio name in two places:
# the header, and as one of the radio button options.
self.assertContains(response, self.portfolio.organization_name, count=2)
self.assertContains(response, self.portfolio.organization_name, count=3)
# We expect the dropdown list to contain the suborganizations that currently exist on this portfolio
self.assertContains(response, self.suborganization.name, count=1)
@ -2298,9 +2298,13 @@ class TestRequestingEntity(WebTest):
form["portfolio_requesting_entity-is_requesting_new_suborganization"] = True
response = form.submit()
self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id)
self.assertContains(response, "Requested suborganization is required.", status_code=200)
self.assertContains(response, "City is required.", status_code=200)
self.assertContains(response, "State, territory, or military post is required.", status_code=200)
self.assertContains(response, "Enter the name of your suborganization.", status_code=200)
self.assertContains(response, "Enter the city where your suborganization is located.", status_code=200)
self.assertContains(
response,
"Select the state, territory, or military post where your suborganization is located.",
status_code=200,
)
@override_flag("organization_feature", active=True)
@override_flag("organization_requests", active=True)

View file

@ -3205,11 +3205,6 @@ class TestDomainRequestWizard(TestWithUser, WebTest):
expected_url = reverse("domain-request:portfolio_requesting_entity", kwargs={"id": domain_request.id})
# This returns the entire url, thus "in"
self.assertIn(expected_url, detail_page.request.url)
# We shouldn't show the "domains" and "domain requests" buttons
# on this page.
self.assertNotContains(detail_page, "Domains")
self.assertNotContains(detail_page, "<span>Domain requests")
else:
self.fail(f"Expected a redirect, but got a different response: {response}")

View file

@ -10,16 +10,38 @@ from registrar.models import (
DomainInformation,
PublicContact,
UserDomainRole,
PortfolioInvitation,
UserGroup,
UserPortfolioPermission,
)
from django.db.models import (
Case,
CharField,
Count,
DateField,
F,
ManyToManyField,
Q,
QuerySet,
TextField,
Value,
When,
OuterRef,
Subquery,
Exists,
Func,
)
from django.db.models import Case, CharField, Count, DateField, F, ManyToManyField, Q, QuerySet, Value, When
from django.utils import timezone
from django.db.models.functions import Concat, Coalesce
from django.contrib.postgres.aggregates import StringAgg
from django.db.models.functions import Concat, Coalesce, Cast
from django.contrib.postgres.aggregates import ArrayAgg, StringAgg
from django.contrib.admin.models import LogEntry, ADDITION
from django.contrib.contenttypes.models import ContentType
from registrar.models.utility.generic_helper import convert_queryset_to_dict
from registrar.models.utility.orm_helper import ArrayRemoveNull
from registrar.models.utility.portfolio_helper import UserPortfolioRoleChoices
from registrar.templatetags.custom_filters import get_region
from registrar.utility.constants import BranchChoices
from registrar.utility.enums import DefaultEmail
from registrar.utility.enums import DefaultEmail, DefaultUserValues
logger = logging.getLogger(__name__)
@ -109,14 +131,14 @@ class BaseExport(ABC):
return Q()
@classmethod
def get_filter_conditions(cls, **export_kwargs):
def get_filter_conditions(cls, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
return Q()
@classmethod
def get_computed_fields(cls):
def get_computed_fields(cls, **kwargs):
"""
Get a dict of computed fields. These are fields that do not exist on the model normally
and will be passed to .annotate() when building a queryset.
@ -145,7 +167,7 @@ class BaseExport(ABC):
return queryset
@classmethod
def write_csv_before(cls, csv_writer, **export_kwargs):
def write_csv_before(cls, csv_writer, **kwargs):
"""
Write to csv file before the write_csv method.
Override in subclasses where needed.
@ -162,7 +184,7 @@ class BaseExport(ABC):
Parameters:
initial_queryset (QuerySet): Initial queryset.
computed_fields (dict, optional): Fields to compute {field_name: expression}.
computed_fields (dict, optional): Fields to compute {field_name: expression}.
related_table_fields (list, optional): Extra fields to retrieve; defaults to annotation keys if None.
include_many_to_many (bool, optional): Determines if we should include many to many fields or not
**kwargs: Additional keyword arguments for specific parameters (e.g., public_contacts, domain_invitations,
@ -192,21 +214,37 @@ class BaseExport(ABC):
return cls.update_queryset(queryset, **kwargs)
@classmethod
def export_data_to_csv(cls, csv_file, **export_kwargs):
def export_data_to_csv(cls, csv_file, **kwargs):
"""
All domain metadata:
Exports domains of all statuses plus domain managers.
"""
writer = csv.writer(csv_file)
columns = cls.get_columns()
models_dict = cls.get_model_annotation_dict(**kwargs)
# Write to csv file before the write_csv
cls.write_csv_before(writer, **kwargs)
# Write the csv file
rows = cls.write_csv(writer, columns, models_dict)
# Return rows that for easier parsing and testing
return rows
@classmethod
def get_annotated_queryset(cls, **kwargs):
"""Returns an annotated queryset based off of all query conditions."""
sort_fields = cls.get_sort_fields()
kwargs = cls.get_additional_args()
# Get additional args and merge with incoming kwargs
additional_args = cls.get_additional_args()
kwargs.update(additional_args)
select_related = cls.get_select_related()
prefetch_related = cls.get_prefetch_related()
exclusions = cls.get_exclusions()
annotations_for_sort = cls.get_annotations_for_sort()
filter_conditions = cls.get_filter_conditions(**export_kwargs)
computed_fields = cls.get_computed_fields()
filter_conditions = cls.get_filter_conditions(**kwargs)
computed_fields = cls.get_computed_fields(**kwargs)
related_table_fields = cls.get_related_table_fields()
model_queryset = (
@ -219,21 +257,11 @@ class BaseExport(ABC):
.order_by(*sort_fields)
.distinct()
)
return cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs)
# Convert the queryset to a dictionary (including annotated fields)
annotated_queryset = cls.annotate_and_retrieve_fields(
model_queryset, computed_fields, related_table_fields, **kwargs
)
models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False)
# Write to csv file before the write_csv
cls.write_csv_before(writer, **export_kwargs)
# Write the csv file
rows = cls.write_csv(writer, columns, models_dict)
# Return rows that for easier parsing and testing
return rows
@classmethod
def get_model_annotation_dict(cls, **kwargs):
return convert_queryset_to_dict(cls.get_annotated_queryset(**kwargs), is_model=False)
@classmethod
def write_csv(
@ -273,6 +301,218 @@ class BaseExport(ABC):
pass
class MemberExport(BaseExport):
"""CSV export for the MembersTable. The members table combines the content
of three tables: PortfolioInvitation, UserPortfolioPermission, and DomainInvitation."""
@classmethod
def model(self):
"""
No model is defined for the member report as it is a combination of multiple fields.
This is a special edge case, but the base report requires this to be defined.
"""
return None
@classmethod
def get_model_annotation_dict(cls, request=None, **kwargs):
"""Combines the permissions and invitation model annotations for
the final returned csv export which combines both of these contexts.
Returns a dictionary of a union between:
- UserPortfolioPermissionModelAnnotation.get_annotated_queryset(portfolio, csv_report=True)
- PortfolioInvitationModelAnnotation.get_annotated_queryset(portfolio, csv_report=True)
"""
portfolio = request.session.get("portfolio")
if not portfolio:
return {}
# Union the two querysets to combine UserPortfolioPermission + invites.
# Unions cannot have a col mismatch, so we must clamp what is returned here.
shared_columns = [
"id",
"first_name",
"last_name",
"email_display",
"last_active",
"roles",
"additional_permissions_display",
"member_display",
"domain_info",
"type",
"joined_date",
"invited_by",
]
# Permissions
permissions = (
UserPortfolioPermission.objects.filter(portfolio=portfolio)
.select_related("user")
.annotate(
first_name=F("user__first_name"),
last_name=F("user__last_name"),
email_display=F("user__email"),
last_active=Coalesce(
Func(F("user__last_login"), Value("YYYY-MM-DD"), function="to_char", output_field=TextField()),
Value("Invalid date"),
output_field=CharField(),
),
additional_permissions_display=F("additional_permissions"),
member_display=Case(
# If email is present and not blank, use email
When(Q(user__email__isnull=False) & ~Q(user__email=""), then=F("user__email")),
# If first name or last name is present, use concatenation of first_name + " " + last_name
When(
Q(user__first_name__isnull=False) | Q(user__last_name__isnull=False),
then=Concat(
Coalesce(F("user__first_name"), Value("")),
Value(" "),
Coalesce(F("user__last_name"), Value("")),
),
),
# If neither, use an empty string
default=Value(""),
output_field=CharField(),
),
domain_info=ArrayAgg(
F("user__permissions__domain__name"),
distinct=True,
# only include domains in portfolio
filter=Q(user__permissions__domain__isnull=False)
& Q(user__permissions__domain__domain_info__portfolio=portfolio),
),
type=Value("member", output_field=CharField()),
joined_date=Func(F("created_at"), Value("YYYY-MM-DD"), function="to_char", output_field=CharField()),
invited_by=cls.get_invited_by_query(object_id_query=cls.get_portfolio_invitation_id_query()),
)
.values(*shared_columns)
)
# Invitations
domain_invitations = DomainInvitation.objects.filter(
email=OuterRef("email"), # Check if email matches the OuterRef("email")
domain__domain_info__portfolio=portfolio, # Check if the domain's portfolio matches the given portfolio
).annotate(domain_info=F("domain__name"))
invitations = (
PortfolioInvitation.objects.exclude(status=PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED)
.filter(portfolio=portfolio)
.annotate(
first_name=Value(None, output_field=CharField()),
last_name=Value(None, output_field=CharField()),
email_display=F("email"),
last_active=Value("Invited", output_field=CharField()),
additional_permissions_display=F("additional_permissions"),
member_display=F("email"),
# Use ArrayRemove to return an empty list when no domain invitations are found
domain_info=ArrayRemoveNull(
ArrayAgg(
Subquery(domain_invitations.values("domain_info")),
distinct=True,
)
),
type=Value("invitedmember", output_field=CharField()),
joined_date=Value("Unretrieved", output_field=CharField()),
invited_by=cls.get_invited_by_query(object_id_query=Cast(OuterRef("id"), output_field=CharField())),
)
.values(*shared_columns)
)
return convert_queryset_to_dict(permissions.union(invitations), is_model=False)
@classmethod
def get_invited_by_query(cls, object_id_query):
"""Returns the user that created the given portfolio invitation.
Grabs this data from the audit log, given that a portfolio invitation object
is specified via object_id_query."""
return Coalesce(
Subquery(
LogEntry.objects.filter(
content_type=ContentType.objects.get_for_model(PortfolioInvitation),
object_id=object_id_query,
action_flag=ADDITION,
)
.annotate(
display_email=Case(
When(
Exists(
UserGroup.objects.filter(
name__in=["cisa_analysts_group", "full_access_group"],
user=OuterRef("user"),
)
),
then=Value(DefaultUserValues.HELP_EMAIL.value),
),
default=F("user__email"),
output_field=CharField(),
)
)
.order_by("action_time")
.values("display_email")[:1]
),
Value(DefaultUserValues.SYSTEM.value),
output_field=CharField(),
)
@classmethod
def get_portfolio_invitation_id_query(cls):
"""Gets the id of the portfolio invitation that created this UserPortfolioPermission.
This makes the assumption that if an invitation is retrieved, it must have created the given
UserPortfolioPermission object."""
return Cast(
Subquery(
PortfolioInvitation.objects.filter(
status=PortfolioInvitation.PortfolioInvitationStatus.RETRIEVED,
# Double outer ref because we first go into the LogEntry query,
# then into the parent UserPortfolioPermission.
email=OuterRef(OuterRef("user__email")),
portfolio=OuterRef(OuterRef("portfolio")),
).values("id")[:1]
),
output_field=CharField(),
)
@classmethod
def get_columns(cls):
"""
Returns the list of column string names for CSV export. Override in subclasses as needed.
"""
return [
"Email",
"Organization admin",
"Invited by",
"Joined date",
"Last active",
"Domain requests",
"Member management",
"Domain management",
"Number of domains",
"Domains",
]
@classmethod
@abstractmethod
def parse_row(cls, columns, model):
"""
Given a set of columns and a model dictionary, generate a new row from cleaned column data.
Must be implemented by subclasses
"""
roles = model.get("roles", [])
permissions = model.get("additional_permissions_display")
user_managed_domains = model.get("domain_info", [])
length_user_managed_domains = len(user_managed_domains)
FIELDS = {
"Email": model.get("email_display"),
"Organization admin": bool(UserPortfolioRoleChoices.ORGANIZATION_ADMIN in roles),
"Invited by": model.get("invited_by"),
"Joined date": model.get("joined_date"),
"Last active": model.get("last_active"),
"Domain requests": UserPortfolioPermission.get_domain_request_permission_display(roles, permissions),
"Member management": UserPortfolioPermission.get_member_permission_display(roles, permissions),
"Domain management": bool(length_user_managed_domains > 0),
"Number of domains": length_user_managed_domains,
"Domains": ",".join(user_managed_domains),
}
return [FIELDS.get(column, "") for column in columns]
class DomainExport(BaseExport):
"""
A collection of functions which return csv files regarding Domains. Although class is
@ -531,10 +771,10 @@ class DomainDataType(DomainExport):
"""
Get a list of tables to pass to prefetch_related when building queryset.
"""
return ["permissions"]
return ["domain__permissions"]
@classmethod
def get_computed_fields(cls, delimiter=", "):
def get_computed_fields(cls, delimiter=", ", **kwargs):
"""
Get a dict of computed fields.
"""
@ -571,7 +811,7 @@ class DomainDataTypeUser(DomainDataType):
"""
@classmethod
def get_filter_conditions(cls, request=None):
def get_filter_conditions(cls, request=None, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
@ -589,7 +829,7 @@ class DomainRequestsDataType:
"""
@classmethod
def get_filter_conditions(cls, request=None):
def get_filter_conditions(cls, request=None, **kwargs):
if request is None or not hasattr(request, "user") or not request.user.is_authenticated:
return Q(id__in=[])
@ -739,7 +979,7 @@ class DomainDataFull(DomainExport):
return ["domain"]
@classmethod
def get_filter_conditions(cls):
def get_filter_conditions(cls, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
@ -751,7 +991,7 @@ class DomainDataFull(DomainExport):
)
@classmethod
def get_computed_fields(cls, delimiter=", "):
def get_computed_fields(cls, delimiter=", ", **kwargs):
"""
Get a dict of computed fields.
"""
@ -833,7 +1073,7 @@ class DomainDataFederal(DomainExport):
return ["domain"]
@classmethod
def get_filter_conditions(cls):
def get_filter_conditions(cls, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
@ -846,7 +1086,7 @@ class DomainDataFederal(DomainExport):
)
@classmethod
def get_computed_fields(cls, delimiter=", "):
def get_computed_fields(cls, delimiter=", ", **kwargs):
"""
Get a dict of computed fields.
"""
@ -930,10 +1170,14 @@ class DomainGrowth(DomainExport):
return ["domain"]
@classmethod
def get_filter_conditions(cls, start_date=None, end_date=None):
def get_filter_conditions(cls, start_date=None, end_date=None, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
if not start_date or not end_date:
# Return nothing
return Q(id__in=[])
filter_ready = Q(
domain__state__in=[Domain.State.READY],
domain__first_ready__gte=start_date,
@ -1002,10 +1246,14 @@ class DomainManaged(DomainExport):
return ["permissions"]
@classmethod
def get_filter_conditions(cls, start_date=None, end_date=None):
def get_filter_conditions(cls, end_date=None, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
if not end_date:
# Return nothing
return Q(id__in=[])
end_date_formatted = format_end_date(end_date)
return Q(
domain__permissions__isnull=False,
@ -1137,10 +1385,14 @@ class DomainUnmanaged(DomainExport):
return ["permissions"]
@classmethod
def get_filter_conditions(cls, start_date=None, end_date=None):
def get_filter_conditions(cls, end_date=None, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
if not end_date:
# Return nothing
return Q(id__in=[])
end_date_formatted = format_end_date(end_date)
return Q(
domain__permissions__isnull=True,
@ -1369,10 +1621,13 @@ class DomainRequestGrowth(DomainRequestExport):
]
@classmethod
def get_filter_conditions(cls, start_date=None, end_date=None):
def get_filter_conditions(cls, start_date=None, end_date=None, **kwargs):
"""
Get a Q object of filter conditions to filter when building queryset.
"""
if not start_date or not end_date:
# Return nothing
return Q(id__in=[])
start_date_formatted = format_start_date(start_date)
end_date_formatted = format_end_date(end_date)
@ -1465,7 +1720,7 @@ class DomainRequestDataFull(DomainRequestExport):
]
@classmethod
def get_computed_fields(cls, delimiter=", "):
def get_computed_fields(cls, delimiter=", ", **kwargs):
"""
Get a dict of computed fields.
"""

View file

@ -35,12 +35,26 @@ class DefaultEmail(Enum):
Overview of emails:
- PUBLIC_CONTACT_DEFAULT: "dotgov@cisa.dhs.gov"
- LEGACY_DEFAULT: "registrar@dotgov.gov"
- HELP_EMAIL: "help@get.gov"
"""
PUBLIC_CONTACT_DEFAULT = "dotgov@cisa.dhs.gov"
LEGACY_DEFAULT = "registrar@dotgov.gov"
class DefaultUserValues(StrEnum):
"""Stores default values for a default user.
Overview of defaults:
- SYSTEM: "System" <= Default username
- UNRETRIEVED: "Unretrieved" <= Default email state
"""
HELP_EMAIL = "help@get.gov"
SYSTEM = "System"
UNRETRIEVED = "Unretrieved"
class Step(StrEnum):
"""
Names for each page of the domain request wizard.

View file

@ -317,15 +317,7 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView):
# Clear context so the prop getter won't create a request here.
# Creating a request will be handled in the post method for the
# intro page.
return render(
request,
"domain_request_intro.html",
{
"hide_requests": True,
"hide_domains": True,
"hide_members": True,
},
)
return render(request, "domain_request_intro.html")
else:
return self.goto(self.steps.first)
@ -487,12 +479,7 @@ class DomainRequestWizard(DomainRequestWizardPermissionView, TemplateView):
"user": self.request.user,
"requested_domain__name": requested_domain_name,
}
# Hides the requests and domains buttons in the navbar
context["hide_requests"] = self.is_portfolio
context["hide_domains"] = self.is_portfolio
context["domain_request_id"] = self.domain_request.id
return context
def get_step_list(self) -> list:

View file

@ -1,7 +1,6 @@
from django.http import JsonResponse
from django.core.paginator import Paginator
from django.db.models import Value, F, CharField, TextField, Q, Case, When, OuterRef, Subquery
from django.db.models.expressions import Func
from django.db.models.functions import Cast, Coalesce, Concat
from django.contrib.postgres.aggregates import ArrayAgg
from django.urls import reverse
@ -12,6 +11,7 @@ from registrar.models.portfolio_invitation import PortfolioInvitation
from registrar.models.user_portfolio_permission import UserPortfolioPermission
from registrar.models.utility.portfolio_helper import UserPortfolioPermissionChoices, UserPortfolioRoleChoices
from registrar.views.utility.mixins import PortfolioMembersPermission
from registrar.models.utility.orm_helper import ArrayRemoveNull
class PortfolioMembersJson(PortfolioMembersPermission, View):
@ -134,7 +134,7 @@ class PortfolioMembersJson(PortfolioMembersPermission, View):
additional_permissions_display=F("additional_permissions"),
member_display=F("email"),
# Use ArrayRemove to return an empty list when no domain invitations are found
domain_info=ArrayRemove(
domain_info=ArrayRemoveNull(
ArrayAgg(
Subquery(domain_invitations.values("domain_info")),
distinct=True,
@ -213,9 +213,3 @@ class PortfolioMembersJson(PortfolioMembersPermission, View):
"svg_icon": ("visibility" if view_only else "settings"),
}
return member_json
# Custom Func to use array_remove to remove null values
class ArrayRemove(Func):
function = "array_remove"
template = "%(function)s(%(expressions)s, NULL)"

View file

@ -169,6 +169,34 @@ class ExportDataTypeUser(View):
return response
class ExportMembersPortfolio(View):
"""Returns a members report for a given portfolio"""
def get(self, request, *args, **kwargs):
"""Returns the members report"""
portfolio = request.session.get("portfolio")
# Check if the user has organization access
if not request.user.is_org_user(request):
return render(request, "403.html", status=403)
# Check if the user has member permissions
if not request.user.has_view_members_portfolio_permission(
portfolio
) and not request.user.has_edit_members_portfolio_permission(portfolio):
return render(request, "403.html", status=403)
# Swap the spaces for dashes to make the formatted name look prettier
portfolio_display = "organization"
if portfolio:
portfolio_display = str(portfolio).lower().replace(" ", "-")
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = f'attachment; filename="members-for-{portfolio_display}.csv"'
csv_export.MemberExport.export_data_to_csv(response, request=request)
return response
class ExportDataTypeRequests(View):
"""Returns a domain requests report for a given user on the request"""