From a7273c8d498384901ed0e37f28d4bd9cf25d31f7 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 20 Jun 2024 14:18:37 -0600 Subject: [PATCH 01/55] Added senior official model (still need to add migrations and foreign key in Portfolios) --- src/registrar/models/senior_official.py | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/registrar/models/senior_official.py diff --git a/src/registrar/models/senior_official.py b/src/registrar/models/senior_official.py new file mode 100644 index 000000000..deb5c0f11 --- /dev/null +++ b/src/registrar/models/senior_official.py @@ -0,0 +1,55 @@ +from django.db import models + +from .utility.time_stamped_model import TimeStampedModel +from phonenumber_field.modelfields import PhoneNumberField # type: ignore + + +class SeniorOfficial(TimeStampedModel): + """ + Senior Official is a distinct Contact-like entity (NOT to be inherited + from Contacts) developed for the unique role these individuals have in + managing Portfolios. + """ + + class Meta: + """Contains meta information about this class""" + + # Placeholder for later tickets... + # indexes = [ + # models.Index(fields=["user"]), + # models.Index(fields=["email"]), + # ] + + + first_name = models.CharField( + null=True, + blank=True, + verbose_name="first name", + ) + last_name = models.CharField( + null=True, + blank=True, + verbose_name="last name", + ) + title = models.CharField( + null=True, + blank=True, + verbose_name="title / role", + ) + phone = PhoneNumberField( + null=True, + blank=True, + ) + + def get_formatted_name(self): + """Returns the contact's name in Western order.""" + names = [n for n in [self.first_name, self.last_name] if n] + return " ".join(names) if names else "Unknown" + + def __str__(self): + if self.first_name or self.last_name: + return self.get_formatted_name() + elif self.pk: + return str(self.pk) + else: + return "" From 625d0e6f1f17d003ae1ccc63535af64ffbb49815 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 20 Jun 2024 15:15:31 -0600 Subject: [PATCH 02/55] Added Senior Official foreign key in Portfolio, along with model field updates (for required fields), and admin.py updates --- src/registrar/admin.py | 25 +++++++++++++++++++++++++ src/registrar/models/__init__.py | 2 ++ src/registrar/models/portfolio.py | 8 ++++++++ src/registrar/models/senior_official.py | 12 ++++++------ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 215239d66..b0a7e0d45 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1016,6 +1016,29 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): # Get the filtered values return super().changelist_view(request, extra_context=extra_context) +class SeniorOfficialAdmin(ListHeaderAdmin, ImportExportModelAdmin): + """Custom Senior Official Admin class.""" + + # NOTE: these are just placeholders. Not part of ACs (haven't been defined yet). Update in future tickets. + search_fields = ["first_name", "last_name"] + search_help_text = "Search by first name or last name." + list_display = [ + "last_name", + ] + + # this ordering effects the ordering of results + # in autocomplete_fields for user + ordering = ["first_name", "last_name"] + + fieldsets = [ + ( + None, + {"fields": ["first_name", "last_name", "title", "phone"]}, + ) + ] + + + class WebsiteResource(resources.ModelResource): """defines how each field in the referenced model should be mapped to the corresponding fields in the @@ -1025,6 +1048,7 @@ class WebsiteResource(resources.ModelResource): model = models.Website + class WebsiteAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom website admin class.""" @@ -2679,6 +2703,7 @@ admin.site.register(models.DomainRequest, DomainRequestAdmin) admin.site.register(models.TransitionDomain, TransitionDomainAdmin) admin.site.register(models.VerifiedByStaff, VerifiedByStaffAdmin) admin.site.register(models.Portfolio, PortfolioAdmin) +admin.site.register(models.SeniorOfficial, SeniorOfficialAdmin) # Register our custom waffle implementations admin.site.register(models.WaffleFlag, WaffleFlagAdmin) diff --git a/src/registrar/models/__init__.py b/src/registrar/models/__init__.py index b2cffaf32..370eb2fa5 100644 --- a/src/registrar/models/__init__.py +++ b/src/registrar/models/__init__.py @@ -17,6 +17,7 @@ from .transition_domain import TransitionDomain from .verified_by_staff import VerifiedByStaff from .waffle_flag import WaffleFlag from .portfolio import Portfolio +from .senior_official import SeniorOfficial __all__ = [ @@ -38,6 +39,7 @@ __all__ = [ "VerifiedByStaff", "WaffleFlag", "Portfolio", + "SeniorOfficial", ] auditlog.register(Contact) diff --git a/src/registrar/models/portfolio.py b/src/registrar/models/portfolio.py index a05422960..931b56405 100644 --- a/src/registrar/models/portfolio.py +++ b/src/registrar/models/portfolio.py @@ -38,6 +38,14 @@ class Portfolio(TimeStampedModel): default=FederalAgency.get_non_federal_agency, ) + senior_official = models.ForeignKey( + "registrar.SeniorOfficial", + on_delete=models.PROTECT, + help_text="Associated senior official", + unique=False, + default=FederalAgency.get_non_federal_agency, + ) + organization_type = models.CharField( max_length=255, choices=OrganizationChoices.choices, diff --git a/src/registrar/models/senior_official.py b/src/registrar/models/senior_official.py index deb5c0f11..001dee579 100644 --- a/src/registrar/models/senior_official.py +++ b/src/registrar/models/senior_official.py @@ -22,18 +22,18 @@ class SeniorOfficial(TimeStampedModel): first_name = models.CharField( - null=True, - blank=True, + null=False, + blank=False, verbose_name="first name", ) last_name = models.CharField( - null=True, - blank=True, + null=False, + blank=False, verbose_name="last name", ) title = models.CharField( - null=True, - blank=True, + null=False, + blank=False, verbose_name="title / role", ) phone = PhoneNumberField( From 1cb6dda7fbadb2b13c74e5e9fd6df155269767f8 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 20 Jun 2024 15:45:04 -0600 Subject: [PATCH 03/55] added migrations --- src/registrar/admin.py | 4 +- ...eniorofficial_portfolio_senior_official.py | 41 +++++++++++++++++++ .../migrations/0106_create_groups_v14.py | 37 +++++++++++++++++ src/registrar/models/__init__.py | 1 + 4 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py create mode 100644 src/registrar/migrations/0106_create_groups_v14.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 54b618731..4f2d2bbe0 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1016,6 +1016,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): # Get the filtered values return super().changelist_view(request, extra_context=extra_context) + class SeniorOfficialAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom Senior Official Admin class.""" @@ -1038,8 +1039,6 @@ class SeniorOfficialAdmin(ListHeaderAdmin, ImportExportModelAdmin): ] - - class WebsiteResource(resources.ModelResource): """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" @@ -1048,7 +1047,6 @@ class WebsiteResource(resources.ModelResource): model = models.Website - class WebsiteAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom website admin class.""" diff --git a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py new file mode 100644 index 000000000..76cc6c0cb --- /dev/null +++ b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2.10 on 2024-06-20 21:16 + +from django.db import migrations, models +import django.db.models.deletion +import phonenumber_field.modelfields +import registrar.models.federal_agency + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0104_create_groups_v13"), + ] + + operations = [ + migrations.CreateModel( + name="SeniorOfficial", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("first_name", models.CharField(verbose_name="first name")), + ("last_name", models.CharField(verbose_name="last name")), + ("title", models.CharField(verbose_name="title / role")), + ( + "phone", + phonenumber_field.modelfields.PhoneNumberField(blank=True, max_length=128, null=True, region=None), + ), + ], + ), + migrations.AddField( + model_name="portfolio", + name="senior_official", + field=models.ForeignKey( + default=registrar.models.federal_agency.FederalAgency.get_non_federal_agency, + help_text="Associated senior official", + on_delete=django.db.models.deletion.PROTECT, + to="registrar.seniorofficial", + ), + ), + ] diff --git a/src/registrar/migrations/0106_create_groups_v14.py b/src/registrar/migrations/0106_create_groups_v14.py new file mode 100644 index 000000000..6e2b38372 --- /dev/null +++ b/src/registrar/migrations/0106_create_groups_v14.py @@ -0,0 +1,37 @@ +# This migration creates the create_full_access_group and create_cisa_analyst_group groups +# It is dependent on 0079 (which populates federal agencies) +# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS +# in the user_group model then: +# [NOT RECOMMENDED] +# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions +# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups +# step 3: fake run the latest migration in the migrations list +# [RECOMMENDED] +# Alternatively: +# step 1: duplicate the migration that loads data +# step 2: docker-compose exec app ./manage.py migrate + +from django.db import migrations +from registrar.models import UserGroup +from typing import Any + + +# For linting: RunPython expects a function reference, +# so let's give it one +def create_groups(apps, schema_editor) -> Any: + UserGroup.create_cisa_analyst_group(apps, schema_editor) + UserGroup.create_full_access_group(apps, schema_editor) + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0105_seniorofficial_portfolio_senior_official"), + ] + + operations = [ + migrations.RunPython( + create_groups, + reverse_code=migrations.RunPython.noop, + atomic=True, + ), + ] diff --git a/src/registrar/models/__init__.py b/src/registrar/models/__init__.py index 370eb2fa5..5d12328e3 100644 --- a/src/registrar/models/__init__.py +++ b/src/registrar/models/__init__.py @@ -60,3 +60,4 @@ auditlog.register(TransitionDomain) auditlog.register(VerifiedByStaff) auditlog.register(WaffleFlag) auditlog.register(Portfolio) +auditlog.register(SeniorOfficial) From 87a9a3f9d60400a341353027a5556b4c9c7c8220 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 21 Jun 2024 13:32:33 -0600 Subject: [PATCH 04/55] model/migration fix --- src/registrar/admin.py | 9 +-------- .../0105_seniorofficial_portfolio_senior_official.py | 9 ++++++--- src/registrar/models/portfolio.py | 3 ++- src/registrar/models/senior_official.py | 10 ---------- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 4f2d2bbe0..53d397667 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1017,7 +1017,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): return super().changelist_view(request, extra_context=extra_context) -class SeniorOfficialAdmin(ListHeaderAdmin, ImportExportModelAdmin): +class SeniorOfficialAdmin(ListHeaderAdmin): """Custom Senior Official Admin class.""" # NOTE: these are just placeholders. Not part of ACs (haven't been defined yet). Update in future tickets. @@ -1031,13 +1031,6 @@ class SeniorOfficialAdmin(ListHeaderAdmin, ImportExportModelAdmin): # in autocomplete_fields for user ordering = ["first_name", "last_name"] - fieldsets = [ - ( - None, - {"fields": ["first_name", "last_name", "title", "phone"]}, - ) - ] - class WebsiteResource(resources.ModelResource): """defines how each field in the referenced model should be mapped to the corresponding fields in the diff --git a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py index 76cc6c0cb..1937039dd 100644 --- a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py @@ -1,9 +1,8 @@ -# Generated by Django 4.2.10 on 2024-06-20 21:16 +# Generated by Django 4.2.10 on 2024-06-21 19:26 from django.db import migrations, models import django.db.models.deletion import phonenumber_field.modelfields -import registrar.models.federal_agency class Migration(migrations.Migration): @@ -27,13 +26,17 @@ class Migration(migrations.Migration): phonenumber_field.modelfields.PhoneNumberField(blank=True, max_length=128, null=True, region=None), ), ], + options={ + "abstract": False, + }, ), migrations.AddField( model_name="portfolio", name="senior_official", field=models.ForeignKey( - default=registrar.models.federal_agency.FederalAgency.get_non_federal_agency, + blank=True, help_text="Associated senior official", + null=True, on_delete=django.db.models.deletion.PROTECT, to="registrar.seniorofficial", ), diff --git a/src/registrar/models/portfolio.py b/src/registrar/models/portfolio.py index d48baa601..b189ccded 100644 --- a/src/registrar/models/portfolio.py +++ b/src/registrar/models/portfolio.py @@ -43,7 +43,8 @@ class Portfolio(TimeStampedModel): on_delete=models.PROTECT, help_text="Associated senior official", unique=False, - default=FederalAgency.get_non_federal_agency, + null=True, + blank=True ) organization_type = models.CharField( diff --git a/src/registrar/models/senior_official.py b/src/registrar/models/senior_official.py index 001dee579..cbe19d873 100644 --- a/src/registrar/models/senior_official.py +++ b/src/registrar/models/senior_official.py @@ -11,16 +11,6 @@ class SeniorOfficial(TimeStampedModel): managing Portfolios. """ - class Meta: - """Contains meta information about this class""" - - # Placeholder for later tickets... - # indexes = [ - # models.Index(fields=["user"]), - # models.Index(fields=["email"]), - # ] - - first_name = models.CharField( null=False, blank=False, From 5ea77e293438c2ef6d7f8f1eda21c33a8d76051b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 21 Jun 2024 14:03:40 -0600 Subject: [PATCH 05/55] linted --- src/registrar/models/portfolio.py | 2 +- src/registrar/models/senior_official.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/portfolio.py b/src/registrar/models/portfolio.py index b189ccded..c72f95c33 100644 --- a/src/registrar/models/portfolio.py +++ b/src/registrar/models/portfolio.py @@ -44,7 +44,7 @@ class Portfolio(TimeStampedModel): help_text="Associated senior official", unique=False, null=True, - blank=True + blank=True, ) organization_type = models.CharField( diff --git a/src/registrar/models/senior_official.py b/src/registrar/models/senior_official.py index cbe19d873..7d1150ab1 100644 --- a/src/registrar/models/senior_official.py +++ b/src/registrar/models/senior_official.py @@ -35,7 +35,7 @@ class SeniorOfficial(TimeStampedModel): """Returns the contact's name in Western order.""" names = [n for n in [self.first_name, self.last_name] if n] return " ".join(names) if names else "Unknown" - + def __str__(self): if self.first_name or self.last_name: return self.get_formatted_name() From 1d0efe9163b8b2ff302f1007b82a1e2a9af03a58 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 25 Jun 2024 14:34:39 -0600 Subject: [PATCH 06/55] Added e-mail field. --- src/registrar/admin.py | 10 ++++------ .../0105_seniorofficial_portfolio_senior_official.py | 3 ++- src/registrar/models/senior_official.py | 5 +++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 53d397667..10157026a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1021,14 +1021,12 @@ class SeniorOfficialAdmin(ListHeaderAdmin): """Custom Senior Official Admin class.""" # NOTE: these are just placeholders. Not part of ACs (haven't been defined yet). Update in future tickets. - search_fields = ["first_name", "last_name"] - search_help_text = "Search by first name or last name." - list_display = [ - "last_name", - ] + search_fields = ["first_name", "last_name", "email"] + search_help_text = "Search by first name, last name or email." + list_display = ["first_name", "last_name", "email"] # this ordering effects the ordering of results - # in autocomplete_fields for user + # in autocomplete_fields for Senior Official ordering = ["first_name", "last_name"] diff --git a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py index 1937039dd..82248c0aa 100644 --- a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-06-21 19:26 +# Generated by Django 4.2.10 on 2024-06-25 20:31 from django.db import migrations, models import django.db.models.deletion @@ -25,6 +25,7 @@ class Migration(migrations.Migration): "phone", phonenumber_field.modelfields.PhoneNumberField(blank=True, max_length=128, null=True, region=None), ), + ("email", models.EmailField(blank=True, max_length=320, null=True)), ], options={ "abstract": False, diff --git a/src/registrar/models/senior_official.py b/src/registrar/models/senior_official.py index 7d1150ab1..3cb064790 100644 --- a/src/registrar/models/senior_official.py +++ b/src/registrar/models/senior_official.py @@ -30,6 +30,11 @@ class SeniorOfficial(TimeStampedModel): null=True, blank=True, ) + email = models.EmailField( + null=True, + blank=True, + max_length=320, + ) def get_formatted_name(self): """Returns the contact's name in Western order.""" From 1c6043fa58a997d7bf23dfa607c3f5cc33a104c2 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 25 Jun 2024 14:43:29 -0600 Subject: [PATCH 07/55] Fixed migrations --- ...niorofficial_portfolio_senior_official.py} | 4 +- .../migrations/0108_create_groups_v14.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0105_seniorofficial_portfolio_senior_official.py => 0107_seniorofficial_portfolio_senior_official.py} (93%) create mode 100644 src/registrar/migrations/0108_create_groups_v14.py diff --git a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py similarity index 93% rename from src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py rename to src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py index 82248c0aa..226822990 100644 --- a/src/registrar/migrations/0105_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-06-25 20:31 +# Generated by Django 4.2.10 on 2024-06-25 20:42 from django.db import migrations, models import django.db.models.deletion @@ -8,7 +8,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0104_create_groups_v13"), + ("registrar", "0106_create_groups_v14"), ] operations = [ diff --git a/src/registrar/migrations/0108_create_groups_v14.py b/src/registrar/migrations/0108_create_groups_v14.py new file mode 100644 index 000000000..e171156fb --- /dev/null +++ b/src/registrar/migrations/0108_create_groups_v14.py @@ -0,0 +1,37 @@ +# This migration creates the create_full_access_group and create_cisa_analyst_group groups +# It is dependent on 0079 (which populates federal agencies) +# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS +# in the user_group model then: +# [NOT RECOMMENDED] +# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions +# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups +# step 3: fake run the latest migration in the migrations list +# [RECOMMENDED] +# Alternatively: +# step 1: duplicate the migration that loads data +# step 2: docker-compose exec app ./manage.py migrate + +from django.db import migrations +from registrar.models import UserGroup +from typing import Any + + +# For linting: RunPython expects a function reference, +# so let's give it one +def create_groups(apps, schema_editor) -> Any: + UserGroup.create_cisa_analyst_group(apps, schema_editor) + UserGroup.create_full_access_group(apps, schema_editor) + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0107_seniorofficial_portfolio_senior_official"), + ] + + operations = [ + migrations.RunPython( + create_groups, + reverse_code=migrations.RunPython.noop, + atomic=True, + ), + ] From 56a8f83e1880946a7c9c12cbad3d8016258d772a Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 26 Jun 2024 17:03:39 -0400 Subject: [PATCH 08/55] init --- src/registrar/models/domain.py | 5 + src/registrar/utility/csv_export.py | 1167 ++++++++++++++------------- src/registrar/views/admin_views.py | 24 +- 3 files changed, 615 insertions(+), 581 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 767227499..7fdc56971 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -151,6 +151,11 @@ class Domain(TimeStampedModel, DomainHelper): # previously existed but has been deleted from the registry DELETED = "deleted", "Deleted" + @classmethod + def get_state_label(cls, state: str): + """Returns the associated label for a given state value""" + return cls(state).label if state else None + @classmethod def get_help_text(cls, state) -> str: """Returns a help message for a desired state. If none is found, an empty string is returned""" diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 1a35c8164..345ed7be1 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from collections import defaultdict import csv import logging from datetime import datetime @@ -32,383 +33,6 @@ def write_header(writer, columns): writer.writerow(columns) -def get_domain_infos(filter_condition, sort_fields): - """ - Returns DomainInformation objects filtered and sorted based on the provided conditions. - filter_condition -> A dictionary of conditions to filter the objects. - sort_fields -> A list of fields to sort the resulting query set. - returns: A queryset of DomainInformation objects - """ - domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - # Do a mass concat of the first and last name fields for authorizing_official. - # The old operation was computationally heavy for some reason, so if we precompute - # this here, it is vastly more efficient. - domain_infos_cleaned = domain_infos.annotate( - ao=Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), - Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), - output_field=CharField(), - ) - ) - return domain_infos_cleaned - - -def parse_row_for_domain( - columns, - domain_info: DomainInformation, - dict_security_emails=None, - should_get_domain_managers=False, - dict_domain_invitations_with_invited_status=None, - dict_user_domain_roles=None, -): - """Given a set of columns, generate a new row from cleaned column data""" - - # Domain should never be none when parsing this information - if domain_info.domain is None: - logger.error("Attemting to parse row for csv exports but Domain is none in a DomainInfo") - raise ValueError("Domain is none") - - domain = domain_info.domain # type: ignore - - # Grab the security email from a preset dictionary. - # If nothing exists in the dictionary, grab from .contacts. - if dict_security_emails is not None and domain.name in dict_security_emails: - _email = dict_security_emails.get(domain.name) - security_email = _email if _email is not None else " " - else: - # If the dictionary doesn't contain that data, lets filter for it manually. - # This is a last resort as this is a more expensive operation. - security_contacts = domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) - _email = security_contacts[0].email if security_contacts else None - security_email = _email if _email is not None else " " - - # These are default emails that should not be displayed in the csv report - invalid_emails = {DefaultEmail.LEGACY_DEFAULT.value, DefaultEmail.PUBLIC_CONTACT_DEFAULT.value} - if security_email.lower() in invalid_emails: - security_email = "(blank)" - - if domain_info.federal_type and domain_info.organization_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{domain_info.get_organization_type_display()} - {domain_info.get_federal_type_display()}" - else: - domain_type = domain_info.get_organization_type_display() - - # create a dictionary of fields which can be included in output - FIELDS = { - "Domain name": domain.name, - "Status": domain.get_state_display(), - "First ready on": domain.first_ready or "(blank)", - "Expiration date": domain.expiration_date or "(blank)", - "Domain type": domain_type, - "Agency": domain_info.federal_agency, - "Organization name": domain_info.organization_name, - "City": domain_info.city, - "State": domain_info.state_territory, - "AO": domain_info.ao, # type: ignore - "AO email": domain_info.authorizing_official.email if domain_info.authorizing_official else " ", - "Security contact email": security_email, - "Created at": domain.created_at, - "Deleted": domain.deleted, - } - - if should_get_domain_managers: - # Get lists of emails for active and invited domain managers - - dms_active_emails = dict_user_domain_roles.get(domain_info.domain.name, []) - dms_invited_emails = dict_domain_invitations_with_invited_status.get(domain_info.domain.name, []) - - # Set up the "matching headers" + row field data for email and status - i = 0 # Declare i outside of the loop to avoid a reference before assignment in the second loop - for i, dm_email in enumerate(dms_active_emails, start=1): - FIELDS[f"Domain manager {i}"] = dm_email - FIELDS[f"DM{i} status"] = "R" - - # Continue enumeration from where we left off and add data for invited domain managers - for j, dm_email in enumerate(dms_invited_emails, start=i + 1): - FIELDS[f"Domain manager {j}"] = dm_email - FIELDS[f"DM{j} status"] = "I" - - row = [FIELDS.get(column, "") for column in columns] - return row - - -def _get_security_emails(sec_contact_ids): - """ - Retrieve security contact emails for the given security contact IDs. - """ - dict_security_emails = {} - public_contacts = ( - PublicContact.objects.only("email", "domain__name") - .select_related("domain") - .filter(registry_id__in=sec_contact_ids) - ) - - # Populate a dictionary of domain names and their security contacts - for contact in public_contacts: - domain: Domain = contact.domain - if domain is not None and domain.name not in dict_security_emails: - dict_security_emails[domain.name] = contact.email - else: - logger.warning("csv_export -> Domain was none for PublicContact") - - return dict_security_emails - - -def count_domain_managers(domain_name, dict_domain_invitations_with_invited_status, dict_user_domain_roles): - """Count active and invited domain managers""" - dms_active = len(dict_user_domain_roles.get(domain_name, [])) - dms_invited = len(dict_domain_invitations_with_invited_status.get(domain_name, [])) - return dms_active, dms_invited - - -def update_columns(columns, dms_total, should_update_columns): - """Update columns if necessary""" - if should_update_columns: - for i in range(1, dms_total + 1): - email_column_header = f"Domain manager {i}" - status_column_header = f"DM{i} status" - if email_column_header not in columns: - columns.append(email_column_header) - columns.append(status_column_header) - should_update_columns = False - return columns, should_update_columns, dms_total - - -def update_columns_with_domain_managers( - columns, - domain_info, - should_update_columns, - dms_total, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, -): - """Helper function to update columns with domain manager information""" - - domain_name = domain_info.domain.name - - try: - dms_active, dms_invited = count_domain_managers( - domain_name, dict_domain_invitations_with_invited_status, dict_user_domain_roles - ) - - if dms_active + dms_invited > dms_total: - dms_total = dms_active + dms_invited - should_update_columns = True - - except Exception as err: - logger.error(f"Exception while parsing domain managers for reports: {err}") - - return update_columns(columns, dms_total, should_update_columns) - - -def build_dictionaries_for_domain_managers(dict_user_domain_roles, dict_domain_invitations_with_invited_status): - """Helper function that builds dicts for invited users and active domain - managers. We do so to avoid filtering within loops.""" - - user_domain_roles = UserDomainRole.objects.all() - - # Iterate through each user domain role and populate the dictionary - for user_domain_role in user_domain_roles: - domain_name = user_domain_role.domain.name - email = user_domain_role.user.email - if domain_name not in dict_user_domain_roles: - dict_user_domain_roles[domain_name] = [] - dict_user_domain_roles[domain_name].append(email) - - domain_invitations_with_invited_status = None - domain_invitations_with_invited_status = DomainInvitation.objects.filter( - status=DomainInvitation.DomainInvitationStatus.INVITED - ).select_related("domain") - - # Iterate through each domain invitation and populate the dictionary - for invite in domain_invitations_with_invited_status: - domain_name = invite.domain.name - email = invite.email - if domain_name not in dict_domain_invitations_with_invited_status: - dict_domain_invitations_with_invited_status[domain_name] = [] - dict_domain_invitations_with_invited_status[domain_name].append(email) - - return dict_user_domain_roles, dict_domain_invitations_with_invited_status - - -def write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, -): - """ - Receives params from the parent methods and outputs a CSV with filtered and sorted domains. - Works with write_header as long as the same writer object is passed. - should_get_domain_managers: Conditional bc we only use domain manager info for export_data_full_to_csv - should_write_header: Conditional bc export_data_domain_growth_to_csv calls write_body twice - """ - - # Retrieve domain information and all sec emails - all_domain_infos = get_domain_infos(filter_condition, sort_fields) - sec_contact_ids = all_domain_infos.values_list("domain__security_contact_registry_id", flat=True) - dict_security_emails = _get_security_emails(sec_contact_ids) - paginator = Paginator(all_domain_infos, 1000) - - # Initialize variables - dms_total = 0 - should_update_columns = False - total_body_rows = [] - dict_user_domain_roles = {} - dict_domain_invitations_with_invited_status = {} - - # Build dictionaries if necessary - if should_get_domain_managers: - dict_user_domain_roles, dict_domain_invitations_with_invited_status = build_dictionaries_for_domain_managers( - dict_user_domain_roles, dict_domain_invitations_with_invited_status - ) - - # Process domain information - for page_num in paginator.page_range: - rows = [] - page = paginator.page(page_num) - for domain_info in page.object_list: - if should_get_domain_managers: - columns, dms_total, should_update_columns = update_columns_with_domain_managers( - columns, - domain_info, - should_update_columns, - dms_total, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, - ) - - try: - row = parse_row_for_domain( - columns, - domain_info, - dict_security_emails, - should_get_domain_managers, - dict_domain_invitations_with_invited_status, - dict_user_domain_roles, - ) - rows.append(row) - except ValueError: - logger.error("csv_export -> Error when parsing row, domain was None") - continue - total_body_rows.extend(rows) - - if should_write_header: - write_header(writer, columns) - writer.writerows(total_body_rows) - - -def export_data_type_to_csv(csv_file): - """ - All domains report with extra columns. - This maps to the "All domain metadata" button. - Exports domains of all statuses. - """ - - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Status", - "First ready on", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - # For domain manager we are pass it in as a parameter below in write_body - ] - - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - write_csv_for_domains( - writer, columns, sort_fields, filter_condition={}, should_get_domain_managers=True, should_write_header=True - ) - - -def export_data_full_to_csv(csv_file): - """All domains report""" - - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - - -def export_data_federal_to_csv(csv_file): - """Federal domains report""" - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) @@ -427,66 +51,6 @@ def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() -def export_data_domain_growth_to_csv(csv_file, start_date, end_date): - """ - Growth report: - Receive start and end dates from the view, parse them. - Request from write_body READY domains that are created between - the start and end dates, as well as DELETED domains that are deleted between - the start and end dates. Specify sort params for both lists. - """ - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - "Created at", - "First ready", - "Deleted", - ] - sort_fields = [ - "domain__first_ready", - "domain__name", - ] - filter_condition = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - "domain__first_ready__gte": start_date_formatted, - } - - # We also want domains deleted between sar and end dates, sorted - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition_for_deleted_domains = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": end_date_formatted, - "domain__deleted__gte": start_date_formatted, - } - - write_csv_for_domains( - writer, columns, sort_fields, filter_condition, should_get_domain_managers=False, should_write_header=True - ) - write_csv_for_domains( - writer, - columns, - sort_fields_for_deleted_domains, - filter_condition_for_deleted_domains, - should_get_domain_managers=False, - should_write_header=False, - ) - - def get_sliced_domains(filter_condition): """Get filtered domains counts sliced by org type and election office. Pass distinct=True when filtering by permissions so we do not to count multiples @@ -558,150 +122,627 @@ def get_sliced_requests(filter_condition): election_board, ] +class DomainExport: + """ + A collection of functions which return csv files regarding the Domain model. + """ -def export_data_managed_domains_to_csv(csv_file, start_date, end_date): - """Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date_formatted, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", + @classmethod + def export_data_type_to_csv(cls, csv_file): + """ + All domain metadata: + Exports domains of all statuses plus domain managers. + """ + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Status", + "First ready on", + "Expiration date", + "Domain type", + "Agency", + "Organization name", "City", - "Special district", - "School district", - "Election office", + "State", + "AO", + "AO email", + "Security contact email", + "Domain managers", + "Invited domain managers", ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant ComainUserRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + domain_infos = ( + DomainInformation.objects.select_related("domain", "authorizing_official") + .prefetch_related("permissions") + .order_by(*sort_fields) + .distinct() + ) + + annotations = cls._domain_metadata_annotations() + + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "authorizing_official__email", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, domain_invitations, user_domain_roles, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_full_to_csv(cls, csv_file): + """Current full""" + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", "City", - "Special district", - "School district", - "Election office", + "State", + "Security contact email", ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } - write_csv_for_domains( + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_federal_to_csv(cls, csv_file): + """Current federal""" + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + # Coalesce is used to replace federal_type of None with ZZZZZ + sort_fields = [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + filter_condition = { + "organization_type__icontains": "federal", + "domain__state__in": [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + } + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Write the csv file + cls.write_csv_for_domains(writer, columns, requests_dict) + + @classmethod + def export_data_domain_growth_to_csv(cls, csv_file, start_date, end_date): + """ + Domain growth: + Receive start and end dates from the view, parse them. + Request from write_body READY domains that are created between + the start and end dates, as well as DELETED domains that are deleted between + the start and end dates. Specify sort params for both lists. + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + "Created at", + "First ready", + "Deleted", + ] + sort_fields = [ + "domain__first_ready", + "domain__name", + ] + filter_condition = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + "domain__first_ready__gte": start_date_formatted, + } + + # We also want domains deleted between sar and end dates, sorted + sort_fields_for_deleted_domains = [ + "domain__deleted", + "domain__name", + ] + filter_condition_for_deleted_domains = { + "domain__state__in": [Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + "domain__deleted__gte": start_date_formatted, + } + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition) + .order_by(*sort_fields) + .distinct() + ) + deleted_domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_condition_for_deleted_domains) + .order_by(*sort_fields_for_deleted_domains) + .distinct() + ) + + annotations = {} + additional_values = [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + ] + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + # Convert the domain request queryset to a dictionary (including annotated fields) + deleted_annotated_domains = cls.annotate_and_retrieve_fields(deleted_domain_infos, annotations, {}, {}, {}, additional_values) + deleted_requests_dict = convert_queryset_to_dict(deleted_annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, columns, requests_dict + + ) + cls.write_csv_for_domains( + writer, + columns, + deleted_requests_dict, + should_write_header=False, + ) + + @classmethod + def export_data_managed_domains_to_csv(cls, csv_file, start_date, end_date): + """ + Managed domains: + Get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + "Domain managers", + "Invited domain managers", + ] + sort_fields = [ + "domain__name", + ] + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_start_date) + writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) + + writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(managed_domains_sliced_at_end_date) + writer.writerow([]) + + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant ComainUserRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + annotations = {} + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + ] + + domain_infos = ( + DomainInformation.objects.select_related("domain") + .prefetch_related("permissions") + .filter(**filter_managed_domains_end_date) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, domain_invitations, user_domain_roles, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, + columns, + requests_dict + ) + + @classmethod + def export_data_unmanaged_domains_to_csv(cls, csv_file, start_date, end_date): + """ + Unmanaged domains: + Get counts for domains that have domain managers for two different dates, + get list of managed domains at end_date.""" + + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + writer = csv.writer(csv_file) + columns = [ + "Domain name", + "Domain type", + ] + sort_fields = [ + "domain__name", + ] + + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + + writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_start_date) + writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + + writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) + writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + writer.writerow(unmanaged_domains_sliced_at_end_date) + writer.writerow([]) + + annotations = {} + # The .values returned from annotate_and_retrieve_fields can't go two levels deep + # (just returns the field id of say, "creator") - so we have to include this. + additional_values = [ + "domain__name", + ] + domain_infos = ( + DomainInformation.objects.select_related("domain") + .filter(**filter_unmanaged_domains_end_date) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the domain request queryset to a dictionary (including annotated fields) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) + + cls.write_csv_for_domains( + writer, + columns, + requests_dict + ) + + @classmethod + def _domain_metadata_annotations(cls, delimiter=", "): + """""" + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def annotate_and_retrieve_fields( + cls, domains, annotations, public_contacts={}, domain_invitations={}, user_domain_roles={}, additional_values=None, include_many_to_many=False + ) -> QuerySet: + """ + Applies annotations to a queryset and retrieves specified fields, + including class-defined and annotation-defined. + + Parameters: + requests (QuerySet): Initial queryset. + annotations (dict, optional): Fields to compute {field_name: expression}. + additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. + include_many_to_many (bool, optional): Determines if we should include many to many fields or not + + Returns: + QuerySet: Contains dictionaries with the specified fields for each record. + """ + + if additional_values is None: + additional_values = [] + + # We can infer that if we're passing in annotations, + # we want to grab the result of said annotation. + if annotations: + additional_values.extend(annotations.keys()) + + # Get prexisting fields on DomainRequest + domain_fields = set() + for field in DomainInformation._meta.get_fields(): + # Exclude many to many fields unless we specify + many_to_many = isinstance(field, ManyToManyField) and include_many_to_many + if many_to_many or not isinstance(field, ManyToManyField): + domain_fields.add(field.name) + + queryset = domains.annotate(**annotations).values(*domain_fields, *additional_values) + annotated_domains = [] + + # Create mapping of domain to a list of invited users and managers + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) + + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) + + # Annotate with security_contact from public_contacts + for domain in queryset: + domain['security_contact_email'] = public_contacts.get(domain.get('domain__registry_id')) + domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) + domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) + annotated_domains.append(domain) + + if annotated_domains: + return annotated_domains + + return queryset + + @staticmethod + def parse_row_for_domains(columns, domain): + """ + Given a set of columns and a request dictionary, generate a new row from cleaned column data. + """ + + status = domain.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = domain.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = domain.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + domain_org_type = domain.get("generic_org_type") + human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) + domain_federal_type = domain.get("federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_federal_type} - {human_readable_domain_org_type}" + + if domain.get("domain__name") == "18f.gov": + print(f'domain_type {domain_type}') + print(f'federal_agency {domain.get("federal_agency")}') + print(f'city {domain.get("city")}') + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + + "Domain name": domain.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": domain.get("federal_agency"), + "Organization name": domain.get("organization_name"), + "City": domain.get("city"), + "State": domain.get("state_territory"), + "AO": domain.get("ao_name"), + "AO email": domain.get("authorizing_official__email"), + "Security contact email": domain.get("security_contact_email"), + "Created at": domain.get("domain__created_at"), + "Deleted": domain.get("domain__deleted"), + "Domain managers": domain.get("managers"), + "Invited domain managers": domain.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + @staticmethod + def write_csv_for_domains( writer, columns, - sort_fields, - filter_managed_domains_end_date, - should_get_domain_managers=True, + domains_dict, should_write_header=True, - ) + ): + """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. + Works with write_header as long as the same writer object is passed.""" + rows = [] + for domain in domains_dict.values(): + try: + row = DomainExport.parse_row_for_domains(columns, domain) + rows.append(row) + except ValueError as err: + logger.error(f"csv_export -> Error when parsing row: {err}") + continue -def export_data_unmanaged_domains_to_csv(csv_file, start_date, end_date): - """Get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" + if should_write_header: + write_header(writer, columns) - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] + writer.writerows(rows) - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date_formatted, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) + # ============================================================= # + # Helper functions for django ORM queries. # + # We are using these rather than pure python for speed reasons. # + # ============================================================= # - writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) + @classmethod + def get_all_security_emails(cls): + """ + Fetch all PublicContact entries and return a mapping of registry_id to email. + """ + public_contacts = PublicContact.objects.values_list('registry_id', 'email') + return {registry_id: email for registry_id, email in public_contacts} + + @classmethod + def get_all_domain_invitations(cls): + """ + Fetch all DomainInvitation entries and return a mapping of domain to email. + """ + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + return list(domain_invitations) - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date_formatted, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) + @classmethod + def get_all_user_domain_roles(cls): + """ + Fetch all UserDomainRole entries and return a mapping of domain to user__email. + """ + user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + return list(user_domain_roles) - writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_unmanaged_domains_end_date, - should_get_domain_managers=False, - should_write_header=True, - ) class DomainRequestExport: diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index f1baa72bd..9b013e36c 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -142,7 +142,7 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.export_data_type_to_csv(response) + csv_export.DomainExport.export_data_type_to_csv(response) return response @@ -151,7 +151,7 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.export_data_full_to_csv(response) + csv_export.DomainExport.export_data_full_to_csv(response) return response @@ -160,7 +160,7 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.export_data_federal_to_csv(response) + csv_export.DomainExport.export_data_federal_to_csv(response) return response @@ -177,31 +177,23 @@ class ExportDomainRequestDataFull(View): class ExportDataDomainsGrowth(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use - # in context to display this data in the template. - csv_export.export_data_domain_growth_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) return response class ExportDataRequestsGrowth(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' - # For #999: set export_data_domain_growth_to_csv to return the resulting queryset, which we can then use - # in context to display this data in the template. csv_export.DomainRequestExport.export_data_requests_growth_to_csv(response, start_date, end_date) return response @@ -209,25 +201,21 @@ class ExportDataRequestsGrowth(View): class ExportDataManagedDomains(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - csv_export.export_data_managed_domains_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) return response class ExportDataUnmanagedDomains(View): def get(self, request, *args, **kwargs): - # Get start_date and end_date from the request's GET parameters - # #999: not needed if we switch to django forms start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - csv_export.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) return response From 4b40b95bac9efeb5ea144a3009f8d41d91f37220 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 26 Jun 2024 17:41:33 -0400 Subject: [PATCH 09/55] fix fed agency and org type displays --- src/registrar/utility/csv_export.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 345ed7be1..2c88f59d2 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -187,6 +187,7 @@ class DomainExport: "domain__created_at", "domain__deleted", "authorizing_official__email", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -234,6 +235,7 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -282,6 +284,7 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -359,6 +362,7 @@ class DomainExport: "domain__expiration_date", "domain__created_at", "domain__deleted", + "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) @@ -658,13 +662,17 @@ class DomainExport: human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) domain_type = human_readable_domain_org_type if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_federal_type} - {human_readable_domain_org_type}" + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" if domain.get("domain__name") == "18f.gov": print(f'domain_type {domain_type}') print(f'federal_agency {domain.get("federal_agency")}') print(f'city {domain.get("city")}') + print(f'agency {domain.get("agency")}') + + print(f'federal_agency__agency {domain.get("federal_agency__agency")}') + # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { @@ -674,7 +682,7 @@ class DomainExport: "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": domain.get("federal_agency"), + "Agency": domain.get("federal_agency__agency"), "Organization name": domain.get("organization_name"), "City": domain.get("city"), "State": domain.get("state_territory"), From c0d513c28f36bb219ed8fa80309883072bbaf977 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 26 Jun 2024 18:31:21 -0400 Subject: [PATCH 10/55] security contact email working --- src/registrar/utility/csv_export.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 2c88f59d2..e51c67f1f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -186,6 +186,7 @@ class DomainExport: "domain__expiration_date", "domain__created_at", "domain__deleted", + "domain__security_contact_registry_id", "authorizing_official__email", "federal_agency__agency", ] @@ -225,6 +226,9 @@ class DomainExport: ], } + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + domain_infos = ( DomainInformation.objects.select_related("domain") .filter(**filter_condition) @@ -235,11 +239,12 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "domain__security_contact_registry_id", "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) # Write the csv file @@ -274,6 +279,9 @@ class DomainExport: ], } + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + domain_infos = ( DomainInformation.objects.select_related("domain") .filter(**filter_condition) @@ -284,11 +292,12 @@ class DomainExport: annotations = {} additional_values = [ "domain__name", + "domain__security_contact_registry_id", "federal_agency__agency", ] # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) + annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) # Write the csv file @@ -629,7 +638,7 @@ class DomainExport: # Annotate with security_contact from public_contacts for domain in queryset: - domain['security_contact_email'] = public_contacts.get(domain.get('domain__registry_id')) + domain['security_contact_email'] = public_contacts.get(domain.get('domain__security_contact_registry_id')) domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) annotated_domains.append(domain) From 72fc09d6d4038a81243895f8e209f16ae1e34bad Mon Sep 17 00:00:00 2001 From: Cameron Dixon Date: Wed, 26 Jun 2024 23:07:11 -0400 Subject: [PATCH 11/55] Update issue-default.yml to use * in links to other issues section Using a * before an issue makes GitHub display the issue details and whether it's open or closed. --- .github/ISSUE_TEMPLATE/issue-default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/issue-default.yml b/.github/ISSUE_TEMPLATE/issue-default.yml index 47fb2c226..f1cb5694f 100644 --- a/.github/ISSUE_TEMPLATE/issue-default.yml +++ b/.github/ISSUE_TEMPLATE/issue-default.yml @@ -31,8 +31,8 @@ body: attributes: label: Links to other issues description: | - "Add issue #numbers this relates to and how (e.g., 🚧 [construction] Blocks, ⛔️ [no_entry] Is blocked by, 🔄 [arrows_counterclockwise] Relates to)." - placeholder: 🔄 Relates to... + "With a `*` to start, add issue #numbers this relates to and how (e.g., 🚧 [construction] Blocks, ⛔️ [no_entry] Is blocked by, 🔄 [arrows_counterclockwise] Relates to)." + placeholder: * 🔄 Relates to... - type: markdown id: note attributes: From cd279468e515622b61ba73b2a16a387453d1710f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 27 Jun 2024 16:15:25 -0400 Subject: [PATCH 12/55] wip; working --- src/registrar/utility/csv_export.py | 996 +++++++++++++++++++++++++++- src/registrar/views/admin_views.py | 36 +- 2 files changed, 1016 insertions(+), 16 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index e51c67f1f..55b0bf37f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod from collections import defaultdict import csv import logging @@ -122,11 +123,1004 @@ def get_sliced_requests(filter_condition): election_board, ] -class DomainExport: +class BaseExport(ABC): + """ + A generic class for exporting data which returns a csv file for the given model. + """ + + @classmethod + @abstractmethod + def model(self): + """ + Property to specify the model that the export class will handle. + Must be implemented by subclasses. + """ + pass + + @classmethod + def get_columns(cls): + """ + Returns the columns for CSV export. Override in subclasses as needed. + """ + return [] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields for the CSV export. Override in subclasses as needed. + """ + return [] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments as an empty dictionary. + Override in subclasses to provide specific arguments. + """ + return {} + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q() + + @classmethod + def get_computed_fields(cls): + """ + Get a dict of computed fields. + """ + return {} + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [] + + @classmethod + def update_queryset(cls, queryset, **kwargs): + """ + Returns an updated queryset. Override in subclass to update queryset. + """ + return queryset + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + Override in subclasses where needed. + """ + pass + + @classmethod + def annotate_and_retrieve_fields( + cls, initial_queryset, computed_fields, related_table_fields=None, include_many_to_many=False, **kwargs + ) -> QuerySet: + """ + Applies annotations to a queryset and retrieves specified fields, + including class-defined and annotation-defined. + + Parameters: + initial_queryset (QuerySet): Initial queryset. + computed_fields (dict, optional): Fields to compute {field_name: expression}. + related_table_fields (list, optional): Extra fields to retrieve; defaults to annotation keys if None. + include_many_to_many (bool, optional): Determines if we should include many to many fields or not + **kwargs: Additional keyword arguments for specific parameters (e.g., public_contacts, domain_invitations, + user_domain_roles). + + Returns: + QuerySet: Contains dictionaries with the specified fields for each record. + """ + if related_table_fields is None: + related_table_fields = [] + + # We can infer that if we're passing in annotations, + # we want to grab the result of said annotation. + if computed_fields: + related_table_fields.extend(computed_fields.keys()) + + # Get prexisting fields on the model + model_fields = set() + for field in cls.model()._meta.get_fields(): + # Exclude many to many fields unless we specify + many_to_many = isinstance(field, ManyToManyField) and include_many_to_many + if many_to_many or not isinstance(field, ManyToManyField): + model_fields.add(field.name) + + queryset = initial_queryset.annotate(**computed_fields).values(*model_fields, *related_table_fields) + + return cls.update_queryset(queryset, **kwargs) + + @classmethod + def export_data_to_csv(cls, csv_file, start_date=None, end_date=None): + """ + All domain metadata: + Exports domains of all statuses plus domain managers. + """ + writer = csv.writer(csv_file) + columns = cls.get_columns() + sort_fields = cls.get_sort_fields() + kwargs = cls.get_additional_args() + select_related = cls.get_select_related() + prefetch_related = cls.get_prefetch_related() + filter_conditions = cls.get_filter_conditions(start_date, end_date) + computed_fields = cls.get_computed_fields() + related_table_fields = cls.get_related_table_fields() + + model_queryset = ( + cls.model().objects.select_related(*select_related) + .prefetch_related(*prefetch_related) + .filter(filter_conditions) + .order_by(*sort_fields) + .distinct() + ) + + # Convert the queryset to a dictionary (including annotated fields) + annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) + models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) + + # Write to csv file before the write_csv + cls.write_csv_before(writer, start_date, end_date) + + # Write the csv file + cls.write_csv(writer, columns, models_dict) + + + @classmethod + def write_csv( + cls, + writer, + columns, + models_dict, + should_write_header=True, + ): + """Receives params from the parent methods and outputs a CSV with filtered and sorted objects. + Works with write_header as long as the same writer object is passed.""" + + rows = [] + for object in models_dict.values(): + try: + row = cls.parse_row(columns, object) + rows.append(row) + except ValueError as err: + logger.error(f"csv_export -> Error when parsing row: {err}") + continue + + if should_write_header: + write_header(writer, columns) + + writer.writerows(rows) + + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + Must be implemented by subclasses + """ + pass + + +class NewDomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. """ + @classmethod + def model(cls): + # Return the model class that this export handles + return DomainInformation + + @classmethod + def update_queryset(cls, queryset, **kwargs): + """ + Returns an updated queryset. + + Add security_contact_email, invited_users, and managers to the queryset, + based on public_contacts, domain_invitations and user_domain_roles + passed through kwargs. + """ + public_contacts = kwargs.get('public_contacts', {}) + domain_invitations = kwargs.get('domain_invitations', {}) + user_domain_roles = kwargs.get('user_domain_roles', {}) + + annotated_domain_infos = [] + + # Create mapping of domain to a list of invited users and managers + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) + + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) + + # Annotate with security_contact from public_contacts + for domain_info in queryset: + domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) + domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) + domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) + annotated_domain_infos.append(domain_info) + + if annotated_domain_infos: + return annotated_domain_infos + + return queryset + + # ============================================================= # + # Helper functions for django ORM queries. # + # We are using these rather than pure python for speed reasons. # + # ============================================================= # + + @classmethod + def get_all_security_emails(cls): + """ + Fetch all PublicContact entries and return a mapping of registry_id to email. + """ + public_contacts = PublicContact.objects.values_list('registry_id', 'email') + return {registry_id: email for registry_id, email in public_contacts} + + @classmethod + def get_all_domain_invitations(cls): + """ + Fetch all DomainInvitation entries and return a mapping of domain to email. + """ + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + return list(domain_invitations) + + @classmethod + def get_all_user_domain_roles(cls): + """ + Fetch all UserDomainRole entries and return a mapping of domain to user__email. + """ + user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + return list(user_domain_roles) + + @classmethod + def parse_row(cls, columns, model): + """ + Given a set of columns and a model dictionary, generate a new row from cleaned column data. + """ + + status = model.get("domain__state") + human_readable_status = Domain.State.get_state_label(status) + + expiration_date = model.get("domain__expiration_date") + if expiration_date is None: + expiration_date = "(blank)" + + first_ready_on = model.get("domain__first_ready") + if first_ready_on is None: + first_ready_on = "(blank)" + + domain_org_type = model.get("generic_org_type") + human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) + domain_federal_type = model.get("federal_type") + human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) + domain_type = human_readable_domain_org_type + if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: + domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" + + if model.get("domain__name") == "18f.gov": + print(f'domain_type {domain_type}') + print(f'federal_agency {model.get("federal_agency")}') + print(f'city {model.get("city")}') + + print(f'agency {model.get("agency")}') + + print(f'federal_agency__agency {model.get("federal_agency__agency")}') + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + + "Domain name": model.get("domain__name"), + "Status": human_readable_status, + "First ready on": first_ready_on, + "Expiration date": expiration_date, + "Domain type": domain_type, + "Agency": model.get("federal_agency__agency"), + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State": model.get("state_territory"), + "AO": model.get("ao_name"), + "AO email": model.get("authorizing_official__email"), + "Security contact email": model.get("security_contact_email"), + "Created at": model.get("domain__created_at"), + "Deleted": model.get("domain__deleted"), + "Domain managers": model.get("managers"), + "Invited domain managers": model.get("invited_users"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + @classmethod + def get_sliced_domains(cls, filter_condition): + """Get filtered domains counts sliced by org type and election office. + Pass distinct=True when filtering by permissions so we do not to count multiples + when a domain has more that one manager. + """ + + domains = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domains.count() + federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() + state_or_territory = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) + tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + special_district = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) + election_board = domains.filter(is_election_board=True).distinct().count() + + return [ + domains_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + +class DomainDataType(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Status", + "First ready on", + "Expiration date", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "AO", + "AO email", + "Security contact email", + "Domain managers", + "Invited domain managers", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant UserDomainRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + return { + 'public_contacts': public_contacts, + 'domain_invitations': domain_invitations, + 'user_domain_roles': user_domain_roles, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain", + "authorizing_official" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + print("DomainDataType::get_related_table_fields") + return [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "domain__security_contact_registry_id", + "authorizing_official__email", + "federal_agency__agency", + ] + + +class DomainDataFull(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + return { + 'public_contacts': public_contacts, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q( + domain__state__in = [ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ], + ) + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__security_contact_registry_id", + "federal_agency__agency", + ] + + +class DomainDataFederal(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Security contact email", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + # Coalesce is used to replace federal_type of None with ZZZZZ + return [ + "organization_type", + Coalesce("federal_type", Value("ZZZZZ")), + "federal_agency", + "domain__name", + ] + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + # Fetch all relevant PublicContact entries + public_contacts = cls.get_all_security_emails() + + return { + 'public_contacts': public_contacts, + } + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + return Q( + organization_type__icontains="federal", + domain__state__in=[ + Domain.State.READY, + Domain.State.DNS_NEEDED, + Domain.State.ON_HOLD, + ] + ) + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ + return { + "ao_name": Concat( + Coalesce(F("authorizing_official__first_name"), Value("")), + Value(" "), + Coalesce(F("authorizing_official__last_name"), Value("")), + output_field=CharField(), + ), + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__security_contact_registry_id", + "federal_agency__agency", + ] + + +class DomainGrowth(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + "Created at", + "First ready", + "Deleted", + ] + + # TODO: The below sort is not working properly + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + '-domain__state', + 'domain__first_ready', + 'domain__deleted', + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + filter_ready = Q( + domain__state__in=[Domain.State.READY], + domain__first_ready__gte=start_date, + domain__first_ready__lte=end_date + ) + filter_deleted = Q( + domain__state__in=[Domain.State.DELETED], + domain__deleted__gte=start_date, + domain__deleted__lte=end_date + ) + return filter_ready | filter_deleted + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + "domain__state", + "domain__first_ready", + "domain__expiration_date", + "domain__created_at", + "domain__deleted", + "federal_agency__agency", + ] + + +class DomainManaged(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + "Domain managers", + "Invited domain managers", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + end_date_formatted = format_end_date(end_date) + return Q( + domain__permissions__isnull=False, + domain__first_ready__lte=end_date_formatted, + ) + + + @classmethod + def get_additional_args(cls): + """ + Returns additional keyword arguments specific to DomainExport. + + Returns: + dict: Dictionary containing public_contacts, domain_invitations, and user_domain_roles. + """ + + # Fetch all relevant Invite entries + domain_invitations = cls.get_all_domain_invitations() + + # Fetch all relevant UserDomainRole entries + user_domain_roles = cls.get_all_user_domain_roles() + + return { + 'domain_invitations': domain_invitations, + 'user_domain_roles': user_domain_roles, + } + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + ] + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + filter_managed_domains_start_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": start_date_formatted, + } + managed_domains_sliced_at_start_date = cls.get_sliced_domains(filter_managed_domains_start_date) + + csv_writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(managed_domains_sliced_at_start_date) + csv_writer.writerow([]) + + filter_managed_domains_end_date = { + "domain__permissions__isnull": False, + "domain__first_ready__lte": end_date_formatted, + } + managed_domains_sliced_at_end_date = cls.get_sliced_domains(filter_managed_domains_end_date) + + csv_writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(managed_domains_sliced_at_end_date) + csv_writer.writerow([]) + + +class DomainUnmanaged(NewDomainExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainExport. + """ + return [ + "Domain name", + "Domain type", + ] + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + 'domain__name', + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "permissions" + ] + + @classmethod + def get_filter_conditions(cls, start_date=None, end_date=None): + """ + Get a Q object of filter conditions to filter when building queryset. + """ + end_date_formatted = format_end_date(end_date) + return Q( + domain__permissions__isnull=True, + domain__first_ready__lte=end_date_formatted, + ) + + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "domain__name", + ] + + @classmethod + def write_csv_before(cls, csv_writer, start_date=None, end_date=None): + """ + Write to csv file before the write_csv method. + + """ + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + filter_unmanaged_domains_start_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": start_date_formatted, + } + unmanaged_domains_sliced_at_start_date = cls.get_sliced_domains(filter_unmanaged_domains_start_date) + + csv_writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(unmanaged_domains_sliced_at_start_date) + csv_writer.writerow([]) + + filter_unmanaged_domains_end_date = { + "domain__permissions__isnull": True, + "domain__first_ready__lte": end_date_formatted, + } + unmanaged_domains_sliced_at_end_date = cls.get_sliced_domains(filter_unmanaged_domains_end_date) + + csv_writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) + csv_writer.writerow( + [ + "Total", + "Federal", + "Interstate", + "State or territory", + "Tribal", + "County", + "City", + "Special district", + "School district", + "Election office", + ] + ) + csv_writer.writerow(unmanaged_domains_sliced_at_end_date) + csv_writer.writerow([]) + + +class DomainExport: @classmethod def export_data_type_to_csv(cls, csv_file): """ diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 9b013e36c..ba57737cb 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,8 +49,8 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, @@ -60,8 +60,8 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -71,8 +71,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.READY], "domain__first_ready__lte": end_date_formatted, } - ready_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_ready_domains_start_date) - ready_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_ready_domains_end_date) + ready_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [models.Domain.State.DELETED], @@ -82,8 +82,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.get_sliced_domains(filter_deleted_domains_start_date) - deleted_domains_sliced_at_end_date = csv_export.get_sliced_domains(filter_deleted_domains_end_date) + deleted_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { "created_at__lte": start_date_formatted, @@ -142,7 +142,8 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - csv_export.DomainExport.export_data_type_to_csv(response) + #csv_export.DomainExport.export_data_type_to_csv(response) + csv_export.DomainDataType.export_data_to_csv(response) return response @@ -151,7 +152,8 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - csv_export.DomainExport.export_data_full_to_csv(response) + #csv_export.DomainExport.export_data_full_to_csv(response) + csv_export.DomainDataFull.export_data_to_csv(response) return response @@ -160,7 +162,8 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - csv_export.DomainExport.export_data_federal_to_csv(response) + #csv_export.DomainExport.export_data_federal_to_csv(response) + csv_export.DomainDataFederal.export_data_to_csv(response) return response @@ -182,7 +185,8 @@ class ExportDataDomainsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) + #csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) + csv_export.DomainGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -205,7 +209,8 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) + #csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) + csv_export.DomainManaged.export_data_to_csv(response, start_date, end_date) return response @@ -215,7 +220,8 @@ class ExportDataUnmanagedDomains(View): start_date = request.GET.get("start_date", "") end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") - response["Content-Disposition"] = f'attachment; filename="unamanaged-domains-{start_date}-to-{end_date}.csv"' - csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + response["Content-Disposition"] = f'attachment; filename="unmanaged-domains-{start_date}-to-{end_date}.csv"' + #csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) + csv_export.DomainUnmanaged.export_data_to_csv(response, start_date, end_date) return response From 10529e40946c1c053ce07deaa8afcfea091450d8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 27 Jun 2024 16:24:50 -0400 Subject: [PATCH 13/55] refactored domain exports --- src/registrar/utility/csv_export.py | 650 +--------------------------- src/registrar/views/admin_views.py | 16 +- 2 files changed, 15 insertions(+), 651 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 55b0bf37f..5a8e88431 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -317,7 +317,7 @@ class BaseExport(ABC): pass -class NewDomainExport(BaseExport): +class DomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. """ @@ -490,7 +490,7 @@ class NewDomainExport(BaseExport): ] -class DomainDataType(NewDomainExport): +class DomainDataType(DomainExport): @classmethod def get_columns(cls): @@ -602,7 +602,7 @@ class DomainDataType(NewDomainExport): ] -class DomainDataFull(NewDomainExport): +class DomainDataFull(DomainExport): @classmethod def get_columns(cls): @@ -695,7 +695,7 @@ class DomainDataFull(NewDomainExport): ] -class DomainDataFederal(NewDomainExport): +class DomainDataFederal(DomainExport): @classmethod def get_columns(cls): @@ -789,7 +789,7 @@ class DomainDataFederal(NewDomainExport): ] -class DomainGrowth(NewDomainExport): +class DomainGrowth(DomainExport): @classmethod def get_columns(cls): @@ -866,7 +866,7 @@ class DomainGrowth(NewDomainExport): ] -class DomainManaged(NewDomainExport): +class DomainManaged(DomainExport): @classmethod def get_columns(cls): @@ -1004,7 +1004,7 @@ class DomainManaged(NewDomainExport): csv_writer.writerow([]) -class DomainUnmanaged(NewDomainExport): +class DomainUnmanaged(DomainExport): @classmethod def get_columns(cls): @@ -1120,642 +1120,6 @@ class DomainUnmanaged(NewDomainExport): csv_writer.writerow([]) -class DomainExport: - @classmethod - def export_data_type_to_csv(cls, csv_file): - """ - All domain metadata: - Exports domains of all statuses plus domain managers. - """ - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Status", - "First ready on", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - "Domain managers", - "Invited domain managers", - ] - - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - # Fetch all relevant Invite entries - domain_invitations = cls.get_all_domain_invitations() - - # Fetch all relevant ComainUserRole entries - user_domain_roles = cls.get_all_user_domain_roles() - - domain_infos = ( - DomainInformation.objects.select_related("domain", "authorizing_official") - .prefetch_related("permissions") - .order_by(*sort_fields) - .distinct() - ) - - annotations = cls._domain_metadata_annotations() - - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - "domain__state", - "domain__first_ready", - "domain__expiration_date", - "domain__created_at", - "domain__deleted", - "domain__security_contact_registry_id", - "authorizing_official__email", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, domain_invitations, user_domain_roles, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_full_to_csv(cls, csv_file): - """Current full""" - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__security_contact_registry_id", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_federal_to_csv(cls, csv_file): - """Current federal""" - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - # Coalesce is used to replace federal_type of None with ZZZZZ - sort_fields = [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", - "domain__name", - ] - filter_condition = { - "organization_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Fetch all relevant PublicContact entries - public_contacts = cls.get_all_security_emails() - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__security_contact_registry_id", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, public_contacts, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Write the csv file - cls.write_csv_for_domains(writer, columns, requests_dict) - - @classmethod - def export_data_domain_growth_to_csv(cls, csv_file, start_date, end_date): - """ - Domain growth: - Receive start and end dates from the view, parse them. - Request from write_body READY domains that are created between - the start and end dates, as well as DELETED domains that are deleted between - the start and end dates. Specify sort params for both lists. - """ - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - "Created at", - "First ready", - "Deleted", - ] - sort_fields = [ - "domain__first_ready", - "domain__name", - ] - filter_condition = { - "domain__state__in": [Domain.State.READY], - "domain__first_ready__lte": end_date_formatted, - "domain__first_ready__gte": start_date_formatted, - } - - # We also want domains deleted between sar and end dates, sorted - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition_for_deleted_domains = { - "domain__state__in": [Domain.State.DELETED], - "domain__deleted__lte": end_date_formatted, - "domain__deleted__gte": start_date_formatted, - } - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition) - .order_by(*sort_fields) - .distinct() - ) - deleted_domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_condition_for_deleted_domains) - .order_by(*sort_fields_for_deleted_domains) - .distinct() - ) - - annotations = {} - additional_values = [ - "domain__name", - "domain__state", - "domain__first_ready", - "domain__expiration_date", - "domain__created_at", - "domain__deleted", - "federal_agency__agency", - ] - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - # Convert the domain request queryset to a dictionary (including annotated fields) - deleted_annotated_domains = cls.annotate_and_retrieve_fields(deleted_domain_infos, annotations, {}, {}, {}, additional_values) - deleted_requests_dict = convert_queryset_to_dict(deleted_annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, columns, requests_dict - - ) - cls.write_csv_for_domains( - writer, - columns, - deleted_requests_dict, - should_write_header=False, - ) - - @classmethod - def export_data_managed_domains_to_csv(cls, csv_file, start_date, end_date): - """ - Managed domains: - Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - "Domain managers", - "Invited domain managers", - ] - sort_fields = [ - "domain__name", - ] - filter_managed_domains_start_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": start_date_formatted, - } - managed_domains_sliced_at_start_date = get_sliced_domains(filter_managed_domains_start_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_start_date) - writer.writerow([]) - - filter_managed_domains_end_date = { - "domain__permissions__isnull": False, - "domain__first_ready__lte": end_date_formatted, - } - managed_domains_sliced_at_end_date = get_sliced_domains(filter_managed_domains_end_date) - - writer.writerow(["MANAGED DOMAINS COUNTS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(managed_domains_sliced_at_end_date) - writer.writerow([]) - - domain_invitations = cls.get_all_domain_invitations() - - # Fetch all relevant ComainUserRole entries - user_domain_roles = cls.get_all_user_domain_roles() - - annotations = {} - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - ] - - domain_infos = ( - DomainInformation.objects.select_related("domain") - .prefetch_related("permissions") - .filter(**filter_managed_domains_end_date) - .order_by(*sort_fields) - .distinct() - ) - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, domain_invitations, user_domain_roles, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, - columns, - requests_dict - ) - - @classmethod - def export_data_unmanaged_domains_to_csv(cls, csv_file, start_date, end_date): - """ - Unmanaged domains: - Get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date.""" - - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - columns = [ - "Domain name", - "Domain type", - ] - sort_fields = [ - "domain__name", - ] - - filter_unmanaged_domains_start_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": start_date_formatted, - } - unmanaged_domains_sliced_at_start_date = get_sliced_domains(filter_unmanaged_domains_start_date) - - writer.writerow(["UNMANAGED DOMAINS AT START DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_start_date) - writer.writerow([]) - - filter_unmanaged_domains_end_date = { - "domain__permissions__isnull": True, - "domain__first_ready__lte": end_date_formatted, - } - unmanaged_domains_sliced_at_end_date = get_sliced_domains(filter_unmanaged_domains_end_date) - - writer.writerow(["UNMANAGED DOMAINS AT END DATE"]) - writer.writerow( - [ - "Total", - "Federal", - "Interstate", - "State or territory", - "Tribal", - "County", - "City", - "Special district", - "School district", - "Election office", - ] - ) - writer.writerow(unmanaged_domains_sliced_at_end_date) - writer.writerow([]) - - annotations = {} - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "domain__name", - ] - domain_infos = ( - DomainInformation.objects.select_related("domain") - .filter(**filter_unmanaged_domains_end_date) - .order_by(*sort_fields) - .distinct() - ) - - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_domains = cls.annotate_and_retrieve_fields(domain_infos, annotations, {}, {}, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_domains, is_model=False) - - cls.write_csv_for_domains( - writer, - columns, - requests_dict - ) - - @classmethod - def _domain_metadata_annotations(cls, delimiter=", "): - """""" - return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), - Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), - output_field=CharField(), - ), - } - - @classmethod - def annotate_and_retrieve_fields( - cls, domains, annotations, public_contacts={}, domain_invitations={}, user_domain_roles={}, additional_values=None, include_many_to_many=False - ) -> QuerySet: - """ - Applies annotations to a queryset and retrieves specified fields, - including class-defined and annotation-defined. - - Parameters: - requests (QuerySet): Initial queryset. - annotations (dict, optional): Fields to compute {field_name: expression}. - additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. - include_many_to_many (bool, optional): Determines if we should include many to many fields or not - - Returns: - QuerySet: Contains dictionaries with the specified fields for each record. - """ - - if additional_values is None: - additional_values = [] - - # We can infer that if we're passing in annotations, - # we want to grab the result of said annotation. - if annotations: - additional_values.extend(annotations.keys()) - - # Get prexisting fields on DomainRequest - domain_fields = set() - for field in DomainInformation._meta.get_fields(): - # Exclude many to many fields unless we specify - many_to_many = isinstance(field, ManyToManyField) and include_many_to_many - if many_to_many or not isinstance(field, ManyToManyField): - domain_fields.add(field.name) - - queryset = domains.annotate(**annotations).values(*domain_fields, *additional_values) - annotated_domains = [] - - # Create mapping of domain to a list of invited users and managers - invited_users_dict = defaultdict(list) - for domain, email in domain_invitations: - invited_users_dict[domain].append(email) - - managers_dict = defaultdict(list) - for domain, email in user_domain_roles: - managers_dict[domain].append(email) - - # Annotate with security_contact from public_contacts - for domain in queryset: - domain['security_contact_email'] = public_contacts.get(domain.get('domain__security_contact_registry_id')) - domain['invited_users'] = ', '.join(invited_users_dict.get(domain.get('domain__name'), [])) - domain['managers'] = ', '.join(managers_dict.get(domain.get('domain__name'), [])) - annotated_domains.append(domain) - - if annotated_domains: - return annotated_domains - - return queryset - - @staticmethod - def parse_row_for_domains(columns, domain): - """ - Given a set of columns and a request dictionary, generate a new row from cleaned column data. - """ - - status = domain.get("domain__state") - human_readable_status = Domain.State.get_state_label(status) - - expiration_date = domain.get("domain__expiration_date") - if expiration_date is None: - expiration_date = "(blank)" - - first_ready_on = domain.get("domain__first_ready") - if first_ready_on is None: - first_ready_on = "(blank)" - - domain_org_type = domain.get("generic_org_type") - human_readable_domain_org_type = DomainRequest.OrganizationChoices.get_org_label(domain_org_type) - domain_federal_type = domain.get("federal_type") - human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) - domain_type = human_readable_domain_org_type - if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: - domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - - if domain.get("domain__name") == "18f.gov": - print(f'domain_type {domain_type}') - print(f'federal_agency {domain.get("federal_agency")}') - print(f'city {domain.get("city")}') - - print(f'agency {domain.get("agency")}') - - print(f'federal_agency__agency {domain.get("federal_agency__agency")}') - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). - FIELDS = { - - "Domain name": domain.get("domain__name"), - "Status": human_readable_status, - "First ready on": first_ready_on, - "Expiration date": expiration_date, - "Domain type": domain_type, - "Agency": domain.get("federal_agency__agency"), - "Organization name": domain.get("organization_name"), - "City": domain.get("city"), - "State": domain.get("state_territory"), - "AO": domain.get("ao_name"), - "AO email": domain.get("authorizing_official__email"), - "Security contact email": domain.get("security_contact_email"), - "Created at": domain.get("domain__created_at"), - "Deleted": domain.get("domain__deleted"), - "Domain managers": domain.get("managers"), - "Invited domain managers": domain.get("invited_users"), - } - - row = [FIELDS.get(column, "") for column in columns] - return row - - @staticmethod - def write_csv_for_domains( - writer, - columns, - domains_dict, - should_write_header=True, - ): - """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. - Works with write_header as long as the same writer object is passed.""" - - rows = [] - for domain in domains_dict.values(): - try: - row = DomainExport.parse_row_for_domains(columns, domain) - rows.append(row) - except ValueError as err: - logger.error(f"csv_export -> Error when parsing row: {err}") - continue - - if should_write_header: - write_header(writer, columns) - - writer.writerows(rows) - - # ============================================================= # - # Helper functions for django ORM queries. # - # We are using these rather than pure python for speed reasons. # - # ============================================================= # - - @classmethod - def get_all_security_emails(cls): - """ - Fetch all PublicContact entries and return a mapping of registry_id to email. - """ - public_contacts = PublicContact.objects.values_list('registry_id', 'email') - return {registry_id: email for registry_id, email in public_contacts} - - @classmethod - def get_all_domain_invitations(cls): - """ - Fetch all DomainInvitation entries and return a mapping of domain to email. - """ - domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') - return list(domain_invitations) - - @classmethod - def get_all_user_domain_roles(cls): - """ - Fetch all UserDomainRole entries and return a mapping of domain to user__email. - """ - user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') - return list(user_domain_roles) - - - class DomainRequestExport: """ A collection of functions which return csv files regarding the DomainRequest model. diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index ba57737cb..9f89c9bd5 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,8 +49,8 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_start_date) - managed_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_managed_domains_end_date) + managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { "domain__permissions__isnull": True, @@ -60,8 +60,8 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) + unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -71,8 +71,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.READY], "domain__first_ready__lte": end_date_formatted, } - ready_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_start_date) - ready_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_ready_domains_end_date) + ready_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_ready_domains_start_date) + ready_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_ready_domains_end_date) filter_deleted_domains_start_date = { "domain__state__in": [models.Domain.State.DELETED], @@ -82,8 +82,8 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_start_date) - deleted_domains_sliced_at_end_date = csv_export.NewDomainExport.get_sliced_domains(filter_deleted_domains_end_date) + deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { "created_at__lte": start_date_formatted, From 38125ea97f0b937160ff9e7041469b8dd6b97884 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 28 Jun 2024 00:24:25 -0400 Subject: [PATCH 14/55] refactored domain request exports --- src/registrar/utility/csv_export.py | 593 ++++++++++++---------------- src/registrar/views/admin_views.py | 18 +- 2 files changed, 268 insertions(+), 343 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 5a8e88431..0940e087f 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -33,17 +33,14 @@ def write_header(writer, columns): """ writer.writerow(columns) - def get_default_start_date(): # Default to a date that's prior to our first deployment return timezone.make_aware(datetime(2023, 11, 1)) - def get_default_end_date(): # Default to now() return timezone.now() - def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() @@ -51,78 +48,6 @@ def format_start_date(start_date): def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() - -def get_sliced_domains(filter_condition): - """Get filtered domains counts sliced by org type and election office. - Pass distinct=True when filtering by permissions so we do not to count multiples - when a domain has more that one manager. - """ - - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() - state_or_territory = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = domains.filter(is_election_board=True).distinct().count() - - return [ - domains_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board, - ] - - -def get_sliced_requests(filter_condition): - """Get filtered requests counts sliced by org type and election office.""" - requests = DomainRequest.objects.all().filter(**filter_condition).distinct() - requests_count = requests.count() - federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() - state_or_territory = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() - ) - tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() - special_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() - ) - school_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() - ) - election_board = requests.filter(is_election_board=True).distinct().count() - - return [ - requests_count, - federal, - interstate, - state_or_territory, - tribal, - county, - city, - special_district, - school_district, - election_board, - ] - class BaseExport(ABC): """ A generic class for exporting data which returns a csv file for the given model. @@ -173,6 +98,13 @@ class BaseExport(ABC): """ return [] + @classmethod + def get_exclusions(cls): + """ + Get a Q object of exclusion conditions to use when building queryset. + """ + return Q() + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -260,6 +192,7 @@ class BaseExport(ABC): kwargs = cls.get_additional_args() select_related = cls.get_select_related() prefetch_related = cls.get_prefetch_related() + exclusions = cls.get_exclusions() filter_conditions = cls.get_filter_conditions(start_date, end_date) computed_fields = cls.get_computed_fields() related_table_fields = cls.get_related_table_fields() @@ -268,6 +201,7 @@ class BaseExport(ABC): cls.model().objects.select_related(*select_related) .prefetch_related(*prefetch_related) .filter(filter_conditions) + .exclude(exclusions) .order_by(*sort_fields) .distinct() ) @@ -282,7 +216,6 @@ class BaseExport(ABC): # Write the csv file cls.write_csv(writer, columns, models_dict) - @classmethod def write_csv( cls, @@ -588,7 +521,6 @@ class DomainDataType(DomainExport): """ Get a list of fields from related tables. """ - print("DomainDataType::get_related_table_fields") return [ "domain__name", "domain__state", @@ -1120,142 +1052,263 @@ class DomainUnmanaged(DomainExport): csv_writer.writerow([]) -class DomainRequestExport: - """ - A collection of functions which return csv files regarding the DomainRequest model. - """ - - # Get all columns on the full metadata report - all_columns = [ - "Domain request", - "Submitted at", - "Status", - "Domain type", - "Federal type", - "Federal agency", - "Organization name", - "Election office", - "City", - "State/territory", - "Region", - "Creator first name", - "Creator last name", - "Creator email", - "Creator approved domains count", - "Creator active requests count", - "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", - "Request purpose", - "Request additional details", - "Other contacts", - "CISA regional representative", - "Current websites", - "Investigator", - ] +class DomainRequestExport(BaseExport): @classmethod - def export_data_requests_growth_to_csv(cls, csv_file, start_date, end_date): + def model(cls): + # Return the model class that this export handles + return DomainRequest + + @classmethod + def get_sliced_requests(cls, filter_condition): + """Get filtered requests counts sliced by org type and election office.""" + requests = DomainRequest.objects.all().filter(**filter_condition).distinct() + requests_count = requests.count() + federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() + interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() + state_or_territory = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + ) + tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() + county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() + city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() + special_district = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + ) + school_district = ( + requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + ) + election_board = requests.filter(is_election_board=True).distinct().count() + + return [ + requests_count, + federal, + interstate, + state_or_territory, + tribal, + county, + city, + special_district, + school_district, + election_board, + ] + + @classmethod + def parse_row(cls, columns, model): """ - Growth report: - Receive start and end dates from the view, parse them. - Request from write_requests_body SUBMITTED requests that are created between - the start and end dates. Specify sort params. + Given a set of columns and a model dictionary, generate a new row from cleaned column data. """ - start_date_formatted = format_start_date(start_date) - end_date_formatted = format_end_date(end_date) - writer = csv.writer(csv_file) - # define columns to include in export - columns = [ + # Handle the federal_type field. Defaults to the wrong format. + federal_type = model.get("federal_type") + human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None + + # Handle the org_type field + org_type = model.get("generic_org_type") or model.get("organization_type") + human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None + + # Handle the status field. Defaults to the wrong format. + status = model.get("status") + status_display = DomainRequest.DomainRequestStatus.get_status_label(status) if status else None + + # Handle the region field. + state_territory = model.get("state_territory") + region = get_region(state_territory) if state_territory else None + + # Handle the requested_domain field (add a default if None) + requested_domain = model.get("requested_domain__name") + requested_domain_name = requested_domain if requested_domain else "No requested domain" + + # Handle the election field. N/A if None, "Yes"/"No" if boolean + human_readable_election_board = "N/A" + is_election_board = model.get("is_election_board") + if is_election_board is not None: + human_readable_election_board = "Yes" if is_election_board else "No" + + # Handle the additional details field. Pipe seperated. + cisa_rep_first = model.get("cisa_representative_first_name") + cisa_rep_last = model.get("cisa_representative_last_name") + name = [n for n in [cisa_rep_first, cisa_rep_last] if n] + + cisa_rep = " ".join(name) if name else None + details = [cisa_rep, model.get("anything_else")] + additional_details = " | ".join([field for field in details if field]) + + # create a dictionary of fields which can be included in output. + # "extra_fields" are precomputed fields (generated in the DB or parsed). + FIELDS = { + # Parsed fields - defined above. + "Domain request": requested_domain_name, + "Region": region, + "Status": status_display, + "Election office": human_readable_election_board, + "Federal type": human_readable_federal_type, + "Domain type": human_readable_org_type, + "Request additional details": additional_details, + # Annotated fields - passed into the request dict. + "Creator approved domains count": model.get("creator_approved_domains_count", 0), + "Creator active requests count": model.get("creator_active_requests_count", 0), + "Alternative domains": model.get("all_alternative_domains"), + "Other contacts": model.get("all_other_contacts"), + "Current websites": model.get("all_current_websites"), + # Untouched FK fields - passed into the request dict. + "Federal agency": model.get("federal_agency__agency"), + "AO first name": model.get("authorizing_official__first_name"), + "AO last name": model.get("authorizing_official__last_name"), + "AO email": model.get("authorizing_official__email"), + "AO title/role": model.get("authorizing_official__title"), + "Creator first name": model.get("creator__first_name"), + "Creator last name": model.get("creator__last_name"), + "Creator email": model.get("creator__email"), + "Investigator": model.get("investigator__email"), + # Untouched fields + "Organization name": model.get("organization_name"), + "City": model.get("city"), + "State/territory": model.get("state_territory"), + "Request purpose": model.get("purpose"), + "CISA regional representative": model.get("cisa_representative_email"), + "Submitted at": model.get("submission_date"), + } + + row = [FIELDS.get(column, "") for column in columns] + return row + + +class DomainRequestGrowth(DomainRequestExport): + + @classmethod + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainRequestGrowth. + """ + return [ "Domain request", "Domain type", "Federal type", "Submitted at", ] - sort_fields = [ + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ "requested_domain__name", ] - filter_condition = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": end_date_formatted, - "submission_date__gte": start_date_formatted, - } - - # We don't want to annotate anything, but we do want to access the requested domain name - annotations = {} - additional_values = ["requested_domain__name"] - - all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() - - annotated_requests = cls.annotate_and_retrieve_fields(all_requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - - cls.write_csv_for_requests(writer, columns, requests_dict) - + @classmethod - def export_full_domain_request_report(cls, csv_file): + def get_filter_conditions(cls, start_date=None, end_date=None): """ - Generates a detailed domain request report to a CSV file. - - Retrieves and annotates DomainRequest objects, excluding 'STARTED' status, - with related data optimizations via select/prefetch and annotation. - - Annotated with counts and aggregates of related entities. - Converts to dict and writes to CSV using predefined columns. - - Parameters: - csv_file (file-like object): Target CSV file. + Get a Q object of filter conditions to filter when building queryset. """ - writer = csv.writer(csv_file) - requests = ( - DomainRequest.objects.select_related( - "creator", "authorizing_official", "federal_agency", "investigator", "requested_domain" - ) - .prefetch_related("current_websites", "other_contacts", "alternative_domains") - .exclude(status__in=[DomainRequest.DomainRequestStatus.STARTED]) - .order_by( - "status", - "requested_domain__name", - ) - .distinct() + start_date_formatted = format_start_date(start_date) + end_date_formatted = format_end_date(end_date) + return Q( + status=DomainRequest.DomainRequestStatus.SUBMITTED, + submission_date__lte=end_date_formatted, + submission_date__gte=start_date_formatted, ) - # Annotations are custom columns returned to the queryset (AKA: computed in the DB). - annotations = cls._full_domain_request_annotations() - - # The .values returned from annotate_and_retrieve_fields can't go two levels deep - # (just returns the field id of say, "creator") - so we have to include this. - additional_values = [ - "requested_domain__name", - "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", - "creator__first_name", - "creator__last_name", - "creator__email", - "investigator__email", + @classmethod + def get_related_table_fields(cls): + """ + Get a list of fields from related tables. + """ + return [ + "requested_domain__name" ] + - # Convert the domain request queryset to a dictionary (including annotated fields) - annotated_requests = cls.annotate_and_retrieve_fields(requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - - # Write the csv file - cls.write_csv_for_requests(writer, cls.all_columns, requests_dict) +class DomainRequestDataFull(DomainRequestExport): @classmethod - def _full_domain_request_annotations(cls, delimiter=" | "): - """Returns the annotations for the full domain request report""" + def get_columns(cls): + """ + Overrides the columns for CSV export specific to DomainRequestGrowth. + """ + return [ + "Domain request", + "Submitted at", + "Status", + "Domain type", + "Federal type", + "Federal agency", + "Organization name", + "Election office", + "City", + "State/territory", + "Region", + "Creator first name", + "Creator last name", + "Creator email", + "Creator approved domains count", + "Creator active requests count", + "Alternative domains", + "AO first name", + "AO last name", + "AO email", + "AO title/role", + "Request purpose", + "Request additional details", + "Other contacts", + "CISA regional representative", + "Current websites", + "Investigator", + ] + + @classmethod + def get_select_related(cls): + """ + Get a list of tables to pass to select_related when building queryset. + """ + return [ + "creator", + "authorizing_official", + "federal_agency", + "investigator", + "requested_domain" + ] + + @classmethod + def get_prefetch_related(cls): + """ + Get a list of tables to pass to prefetch_related when building queryset. + """ + return [ + "current_websites", + "other_contacts", + "alternative_domains" + ] + + @classmethod + def get_exclusions(cls): + """ + Get a Q object of exclusion conditions to use when building queryset. + """ + return Q( + status__in=[DomainRequest.DomainRequestStatus.STARTED] + ) + + @classmethod + def get_sort_fields(cls): + """ + Returns the sort fields. + """ + return [ + "status", + "requested_domain__name", + ] + + @classmethod + def get_computed_fields(cls, delimiter=", "): + """ + Get a dict of computed fields. + """ return { - "creator_approved_domains_count": DomainRequestExport.get_creator_approved_domains_count_query(), - "creator_active_requests_count": DomainRequestExport.get_creator_active_requests_count_query(), + "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), + "creator_active_requests_count": cls.get_creator_active_requests_count_query(), "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), "all_alternative_domains": StringAgg("alternative_domains__website", delimiter=delimiter, distinct=True), # Coerce the other contacts object to "{first_name} {last_name} {email}" @@ -1271,155 +1324,33 @@ class DomainRequestExport: distinct=True, ), } - - @staticmethod - def write_csv_for_requests( - writer, - columns, - requests_dict, - should_write_header=True, - ): - """Receives params from the parent methods and outputs a CSV with filtered and sorted requests. - Works with write_header as long as the same writer object is passed.""" - - rows = [] - for request in requests_dict.values(): - try: - row = DomainRequestExport.parse_row_for_requests(columns, request) - rows.append(row) - except ValueError as err: - logger.error(f"csv_export -> Error when parsing row: {err}") - continue - - if should_write_header: - write_header(writer, columns) - - writer.writerows(rows) - - @staticmethod - def parse_row_for_requests(columns, request): - """ - Given a set of columns and a request dictionary, generate a new row from cleaned column data. - """ - - # Handle the federal_type field. Defaults to the wrong format. - federal_type = request.get("federal_type") - human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None - - # Handle the org_type field - org_type = request.get("generic_org_type") or request.get("organization_type") - human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None - - # Handle the status field. Defaults to the wrong format. - status = request.get("status") - status_display = DomainRequest.DomainRequestStatus.get_status_label(status) if status else None - - # Handle the region field. - state_territory = request.get("state_territory") - region = get_region(state_territory) if state_territory else None - - # Handle the requested_domain field (add a default if None) - requested_domain = request.get("requested_domain__name") - requested_domain_name = requested_domain if requested_domain else "No requested domain" - - # Handle the election field. N/A if None, "Yes"/"No" if boolean - human_readable_election_board = "N/A" - is_election_board = request.get("is_election_board") - if is_election_board is not None: - human_readable_election_board = "Yes" if is_election_board else "No" - - # Handle the additional details field. Pipe seperated. - cisa_rep_first = request.get("cisa_representative_first_name") - cisa_rep_last = request.get("cisa_representative_last_name") - name = [n for n in [cisa_rep_first, cisa_rep_last] if n] - - cisa_rep = " ".join(name) if name else None - details = [cisa_rep, request.get("anything_else")] - additional_details = " | ".join([field for field in details if field]) - - # create a dictionary of fields which can be included in output. - # "extra_fields" are precomputed fields (generated in the DB or parsed). - FIELDS = { - # Parsed fields - defined above. - "Domain request": requested_domain_name, - "Region": region, - "Status": status_display, - "Election office": human_readable_election_board, - "Federal type": human_readable_federal_type, - "Domain type": human_readable_org_type, - "Request additional details": additional_details, - # Annotated fields - passed into the request dict. - "Creator approved domains count": request.get("creator_approved_domains_count", 0), - "Creator active requests count": request.get("creator_active_requests_count", 0), - "Alternative domains": request.get("all_alternative_domains"), - "Other contacts": request.get("all_other_contacts"), - "Current websites": request.get("all_current_websites"), - # Untouched FK fields - passed into the request dict. - "Federal agency": request.get("federal_agency__agency"), - "AO first name": request.get("authorizing_official__first_name"), - "AO last name": request.get("authorizing_official__last_name"), - "AO email": request.get("authorizing_official__email"), - "AO title/role": request.get("authorizing_official__title"), - "Creator first name": request.get("creator__first_name"), - "Creator last name": request.get("creator__last_name"), - "Creator email": request.get("creator__email"), - "Investigator": request.get("investigator__email"), - # Untouched fields - "Organization name": request.get("organization_name"), - "City": request.get("city"), - "State/territory": request.get("state_territory"), - "Request purpose": request.get("purpose"), - "CISA regional representative": request.get("cisa_representative_email"), - "Submitted at": request.get("submission_date"), - } - - row = [FIELDS.get(column, "") for column in columns] - return row - + @classmethod - def annotate_and_retrieve_fields( - cls, requests, annotations, additional_values=None, include_many_to_many=False - ) -> QuerySet: + def get_related_table_fields(cls): """ - Applies annotations to a queryset and retrieves specified fields, - including class-defined and annotation-defined. - - Parameters: - requests (QuerySet): Initial queryset. - annotations (dict, optional): Fields to compute {field_name: expression}. - additional_values (list, optional): Extra fields to retrieve; defaults to annotation keys if None. - include_many_to_many (bool, optional): Determines if we should include many to many fields or not - - Returns: - QuerySet: Contains dictionaries with the specified fields for each record. + Get a list of fields from related tables. """ - - if additional_values is None: - additional_values = [] - - # We can infer that if we're passing in annotations, - # we want to grab the result of said annotation. - if annotations: - additional_values.extend(annotations.keys()) - - # Get prexisting fields on DomainRequest - domain_request_fields = set() - for field in DomainRequest._meta.get_fields(): - # Exclude many to many fields unless we specify - many_to_many = isinstance(field, ManyToManyField) and include_many_to_many - if many_to_many or not isinstance(field, ManyToManyField): - domain_request_fields.add(field.name) - - queryset = requests.annotate(**annotations).values(*domain_request_fields, *additional_values) - return queryset + return [ + "requested_domain__name", + "federal_agency__agency", + "authorizing_official__first_name", + "authorizing_official__last_name", + "authorizing_official__email", + "authorizing_official__title", + "creator__first_name", + "creator__last_name", + "creator__email", + "investigator__email", + ] + # ============================================================= # # Helper functions for django ORM queries. # # We are using these rather than pure python for speed reasons. # # ============================================================= # - @staticmethod - def get_creator_approved_domains_count_query(): + @classmethod + def get_creator_approved_domains_count_query(cls): """ Generates a Count query for distinct approved domain requests per creator. @@ -1434,8 +1365,8 @@ class DomainRequestExport: ) return query - @staticmethod - def get_creator_active_requests_count_query(): + @classmethod + def get_creator_active_requests_count_query(cls): """ Generates a Count query for distinct approved domain requests per creator. diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 9f89c9bd5..63e8492b5 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -91,8 +91,8 @@ class AnalyticsView(View): filter_requests_end_date = { "created_at__lte": end_date_formatted, } - requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_requests_start_date) - requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_requests_end_date) + requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_requests_start_date) + requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_requests_end_date) filter_submitted_requests_start_date = { "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, @@ -102,8 +102,8 @@ class AnalyticsView(View): "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) + submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) context = dict( # Generate a dictionary of context variables that are common across all admin templates @@ -142,7 +142,6 @@ class ExportDataType(View): # match the CSV example with all the fields response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="domains-by-type.csv"' - #csv_export.DomainExport.export_data_type_to_csv(response) csv_export.DomainDataType.export_data_to_csv(response) return response @@ -152,7 +151,6 @@ class ExportDataFull(View): # Smaller export based on 1 response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full.csv"' - #csv_export.DomainExport.export_data_full_to_csv(response) csv_export.DomainDataFull.export_data_to_csv(response) return response @@ -162,7 +160,6 @@ class ExportDataFederal(View): # Federal only response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-federal.csv"' - #csv_export.DomainExport.export_data_federal_to_csv(response) csv_export.DomainDataFederal.export_data_to_csv(response) return response @@ -174,7 +171,7 @@ class ExportDomainRequestDataFull(View): """Returns a content disposition response for current-full-domain-request.csv""" response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="current-full-domain-request.csv"' - csv_export.DomainRequestExport.export_full_domain_request_report(response) + csv_export.DomainRequestDataFull.export_data_to_csv(response) return response @@ -185,7 +182,6 @@ class ExportDataDomainsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_domain_growth_to_csv(response, start_date, end_date) csv_export.DomainGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -198,7 +194,7 @@ class ExportDataRequestsGrowth(View): response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="requests-{start_date}-to-{end_date}.csv"' - csv_export.DomainRequestExport.export_data_requests_growth_to_csv(response, start_date, end_date) + csv_export.DomainRequestGrowth.export_data_to_csv(response, start_date, end_date) return response @@ -209,7 +205,6 @@ class ExportDataManagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="managed-domains-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_managed_domains_to_csv(response, start_date, end_date) csv_export.DomainManaged.export_data_to_csv(response, start_date, end_date) return response @@ -221,7 +216,6 @@ class ExportDataUnmanagedDomains(View): end_date = request.GET.get("end_date", "") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f'attachment; filename="unmanaged-domains-{start_date}-to-{end_date}.csv"' - #csv_export.DomainExport.export_data_unmanaged_domains_to_csv(response, start_date, end_date) csv_export.DomainUnmanaged.export_data_to_csv(response, start_date, end_date) return response From 59e912b69a8fa3e437d8249c50a24bcc1e4c51e8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 28 Jun 2024 00:55:42 -0400 Subject: [PATCH 15/55] sorting in domain growth report --- src/registrar/utility/csv_export.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 0940e087f..6875f0e3d 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -12,6 +12,7 @@ from registrar.models import ( UserDomainRole, ) from django.db.models import QuerySet, Value, CharField, Count, Q, F +from django.db.models import Case, When, DateField from django.db.models import ManyToManyField from django.utils import timezone from django.core.paginator import Paginator @@ -119,6 +120,13 @@ class BaseExport(ABC): """ return {} + @classmethod + def get_annotations_for_sort(cls): + """ + Get a dict of annotations to make available for order_by clause. + """ + return {} + @classmethod def get_related_table_fields(cls): """ @@ -193,6 +201,7 @@ class BaseExport(ABC): select_related = cls.get_select_related() prefetch_related = cls.get_prefetch_related() exclusions = cls.get_exclusions() + annotations_for_sort = cls.get_annotations_for_sort() filter_conditions = cls.get_filter_conditions(start_date, end_date) computed_fields = cls.get_computed_fields() related_table_fields = cls.get_related_table_fields() @@ -202,6 +211,7 @@ class BaseExport(ABC): .prefetch_related(*prefetch_related) .filter(filter_conditions) .exclude(exclusions) + .annotate(**annotations_for_sort) .order_by(*sort_fields) .distinct() ) @@ -742,7 +752,20 @@ class DomainGrowth(DomainExport): "Deleted", ] - # TODO: The below sort is not working properly + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + today = timezone.now().date() + return { + "custom_sort": Case( + When(domain__state=Domain.State.READY, then='domain__first_ready'), + When(domain__state=Domain.State.DELETED, then='domain__deleted'), + default=Value(today), # Default value if no conditions match + output_field=DateField() + ) + } @classmethod def get_sort_fields(cls): @@ -751,8 +774,7 @@ class DomainGrowth(DomainExport): """ return [ '-domain__state', - 'domain__first_ready', - 'domain__deleted', + 'custom_sort', 'domain__name', ] From b600a26eb86afd3ae3f1c930dfb195db2e1728b2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Fri, 28 Jun 2024 16:34:39 -0400 Subject: [PATCH 16/55] Unit tests --- .../generate_current_federal_report.py | 2 +- .../commands/generate_current_full_report.py | 2 +- src/registrar/tests/test_reports.py | 926 +++++++----------- src/registrar/utility/csv_export.py | 284 +++--- src/registrar/views/admin_views.py | 24 +- 5 files changed, 534 insertions(+), 704 deletions(-) diff --git a/src/registrar/management/commands/generate_current_federal_report.py b/src/registrar/management/commands/generate_current_federal_report.py index 6516bf99b..97d4fd7e4 100644 --- a/src/registrar/management/commands/generate_current_federal_report.py +++ b/src/registrar/management/commands/generate_current_federal_report.py @@ -50,7 +50,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_federal_to_csv(file) + csv_export.DomainDataFederal.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/management/commands/generate_current_full_report.py b/src/registrar/management/commands/generate_current_full_report.py index be810ee10..4bcb9f502 100644 --- a/src/registrar/management/commands/generate_current_full_report.py +++ b/src/registrar/management/commands/generate_current_full_report.py @@ -49,7 +49,7 @@ class Command(BaseCommand): # Generate a file locally for upload with open(file_path, "w") as file: - csv_export.export_data_full_to_csv(file) + csv_export.DomainDataFull.export_data_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 0028034fb..0015ae84f 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -1,21 +1,23 @@ -import csv import io from django.test import Client, RequestFactory from io import StringIO from registrar.models.domain_request import DomainRequest from registrar.models.domain import Domain -from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.utility.csv_export import ( - export_data_managed_domains_to_csv, - export_data_unmanaged_domains_to_csv, - get_sliced_domains, - get_sliced_requests, - write_csv_for_domains, + DomainDataFull, + DomainDataType, + DomainDataFederal, + DomainGrowth, + DomainManaged, + DomainUnmanaged, + DomainExport, + DomainRequestExport, + DomainRequestGrowth, + DomainRequestDataFull, get_default_start_date, get_default_end_date, - DomainRequestExport, ) - +from django.db.models import Case, When from django.core.management import call_command from unittest.mock import MagicMock, call, mock_open, patch from api.views import get_current_federal, get_current_full @@ -45,10 +47,10 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -67,11 +69,12 @@ class CsvReportsTest(MockDb): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), - call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), - call("adomain10.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), - call("adomain2.gov,Interstate,,,,, \r\n"), + call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\r\n"), + call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\r\n"), + call("adomain2.gov,Interstate,,,,,\r\n"), + call("zdomain12.gov,Interstate,,,,,\r\n"), ] # We don't actually want to write anything for a test case, # we just want to verify what is being written. @@ -202,494 +205,299 @@ class ExportDataTest(MockDb, MockEppLib): def tearDown(self): super().tearDown() - def test_export_domains_to_writer_security_emails_and_first_ready(self): - """Test that export_domains_to_writer returns the - expected security email and first_ready value""" + @less_console_noise_decorator + def test_domain_data_type(self): + """Shows security contacts, domain managers, ao""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataType.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO," + "AO email,Security contact email,Domain managers,Invited domain managers\n" + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," + "meoward@rocks.com,\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," + ', ,,dotgov@cisa.dhs.gov,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,,," + "squeaker@rocks.com\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,, ,," + "security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,,,\n" + "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,registrar@dotgov.gov," + "meoward@rocks.com,squeaker@rocks.com\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): - # Add security email information - self.domain_1.name = "defaultsecurity.gov" - self.domain_1.save() - # Invoke setter - self.domain_1.security_contact - # Invoke setter - self.domain_2.security_contact - # Invoke setter - self.domain_3.security_contact + @less_console_noise_decorator + def test_domain_data_full(self): + """Shows security contacts, filtered by state""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "adomain2.gov,Interstate,,,,,registrar@dotgov.gov\n" + "zdomain12.gov,Interstate,,,,,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Add a first ready date on the first domain. Leaving the others blank. - self.domain_1.first_ready = get_default_start_date() - self.domain_1.save() + @less_console_noise_decorator + def test_domain_data_federal(self): + """Shows security contacts, filtered by state and org type""" + # Add security email information + self.domain_1.name = "defaultsecurity.gov" + self.domain_1.save() + # Invoke setter + self.domain_1.security_contact + # Invoke setter + self.domain_2.security_contact + # Invoke setter + self.domain_3.security_contact + # Add a first ready date on the first domain. Leaving the others blank. + self.domain_1.first_ready = get_default_start_date() + self.domain_1.save() + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainDataFederal.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains, + # sorted alphabetially by domain name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" + "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,\n" + "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,dotgov@cisa.dhs.gov\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - "Status", - "Expiration date", - "First ready on", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, + @less_console_noise_decorator + def test_domain_growth(self): + """Shows ready and deleted domains within a date range, sorted""" + # Remove "Created at" and "First ready" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + # "Created at", + # "First ready", + "Deleted", + ] + sort = { + "custom_sort": Case( + When(domain__state=Domain.State.READY, then="domain__created_at"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), ) + } + with patch("registrar.utility.csv_export.DomainGrowth.get_columns", return_value=columns): + with patch("registrar.utility.csv_export.DomainGrowth.get_annotations_for_sort", return_value=sort): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name + # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City," + "State,Status,Expiration date, Deleted\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" + "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" + "zdomain12.govInterstateReady(blank)\n" + "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = ( + csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + ) + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Security contact email,Status,Expiration date, First ready on\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready,(blank),2024-04-03\n" - "adomain2.gov,Interstate,(blank),Dns needed,(blank),(blank)\n" - "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank),2024-04-02\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,security@mail.gov,On hold,2023-11-15,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission," - "(blank),Ready,(blank),2023-11-01\n" - "zdomain12.govInterstateReady,(blank),2024-04-02\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_csv_for_domains(self): - """Test that write_body returns the - existing domain, test that sort by domain name works, - test that filter works""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Submitter", - "Submitter title", - "Submitter email", - "Submitter phone", - "Security contact email", - "Status", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,AO," - "AO email,Submitter,Submitter title,Submitter email,Submitter phone," - "Security contact email,Status\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" - "adomain2.gov,Interstate,Dns needed\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n" - "zdomain12.govInterstateReady\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_additional(self): - """An additional test for filters and multi-column sort""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Security contact email", - ] - sort_fields = ["domain__name", "federal_agency", "generic_org_type"] - filter_condition = { - "generic_org_type__icontains": "federal", - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains, - # federal only - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Security contact email\n" - "adomain10.gov,Federal,Armed Forces Retirement Home\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommission\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_write_domains_body_with_date_filter_pulls_domains_in_range(self): - """Test that domains that are - 1. READY and their first_ready dates are in range - 2. DELETED and their deleted dates are in range - are pulled when the growth report conditions are applied to export_domains_to_writed. - Test that ready domains are sorted by first_ready/deleted dates first, names second. - - We considered testing export_data_domain_growth_to_csv which calls write_body - and would have been easy to set up, but expected_content would contain created_at dates - which are hard to mock. - - TODO: Simplify if created_at is not needed for the report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "Status", - "Expiration date", - ] - sort_fields = [ - "created_at", - "domain__name", - ] - sort_fields_for_deleted_domains = [ - "domain__deleted", - "domain__name", - ] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - ], - "domain__first_ready__lte": self.end_date, - "domain__first_ready__gte": self.start_date, - } - filter_conditions_for_deleted_domains = { - "domain__state__in": [ - Domain.State.DELETED, - ], - "domain__deleted__lte": self.end_date, - "domain__deleted__gte": self.start_date, - } - - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=False, - should_write_header=True, - ) - write_csv_for_domains( - writer, - columns, - sort_fields_for_deleted_domains, - filter_conditions_for_deleted_domains, - should_get_domain_managers=False, - should_write_header=False, - ) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - - # Read the content into a variable - csv_content = csv_file.read() - - # We expect READY domains first, created between day-2 and day+2, sorted by created_at then name - # and DELETED domains deleted between day-2 and day+2, sorted by deleted then name - expected_content = ( - "Domain name,Domain type,Agency,Organization name,City," - "State,Status,Expiration date\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" - "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank)\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank)\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank)\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_domains_to_writer_domain_managers(self): - """Test that export_domains_to_writer returns the - expected domain managers. + @less_console_noise_decorator + def test_domain_managed(self): + """Shows ready and deleted domains by an end date, sorted An invited user, woofwardthethird, should also be pulled into this report. squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). - She should show twice in this report but not in test_export_data_managed_domains_to_csv.""" + She should show twice in this report but not in test_DomainManaged.""" + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainManaged.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), + ) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "MANAGED DOMAINS COUNTS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "MANAGED DOMAINS COUNTS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City," + "Special district,School district,Election office\n" + "3,2,1,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type,Domain managers,Invited domain managers\n" + "cdomain11.gov,Federal - Executive,meoward@rocks.com,\n" + 'cdomain1.gov,Federal - Executive,"meoward@rocks.com, info@example.com, big_lebowski@dude.co",' + "woofwardthethird@rocks.com\n" + "zdomain12.gov,Interstate,meoward@rocks.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) - with less_console_noise(): + @less_console_noise_decorator + def test_domain_unmanaged(self): + """Shows unmanaged domains by an end date, sorted""" + # Create a CSV file in memory + csv_file = StringIO() + DomainUnmanaged.export_data_to_csv( + csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + + # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. + expected_content = ( + "UNMANAGED DOMAINS AT START DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "0,0,0,0,0,0,0,0,0,0\n" + "\n" + "UNMANAGED DOMAINS AT END DATE\n" + "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," + "School district,Election office\n" + "1,1,0,0,0,0,0,0,0,0\n" + "\n" + "Domain name,Domain type\n" + "adomain10.gov,Federal\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + @less_console_noise_decorator + def test_domain_request_growth(self): + """Shows submitted requests within a date range, sorted""" + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + "Domain type", + "Federal type", + # "Submitted at", + ] + with patch("registrar.utility.csv_export.DomainRequestGrowth.get_columns", return_value=columns): # Create a CSV file in memory csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - columns = [ - "Domain name", - "Status", - "Expiration date", - "Domain type", - "Agency", - "Organization name", - "City", - "State", - "AO", - "AO email", - "Security contact email", - ] - sort_fields = ["domain__name"] - filter_condition = { - "domain__state__in": [ - Domain.State.READY, - Domain.State.DNS_NEEDED, - Domain.State.ON_HOLD, - ], - } - # Call the export functions - write_csv_for_domains( - writer, - columns, - sort_fields, - filter_condition, - should_get_domain_managers=True, - should_write_header=True, + DomainRequestGrowth.export_data_to_csv( + csv_file, + self.start_date.strftime("%Y-%m-%d"), + self.end_date.strftime("%Y-%m-%d"), ) - # Reset the CSV file's position to the beginning csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() - # We expect READY domains, - # sorted alphabetially by domain name - expected_content = ( - "Domain name,Status,Expiration date,Domain type,Agency," - "Organization name,City,State,AO,AO email," - "Security contact email,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "adomain10.gov,Ready,(blank),Federal,Armed Forces Retirement Home,,,, , ,squeaker@rocks.com, I\n" - "adomain2.gov,Dns needed,(blank),Interstate,,,,, , , ,meoward@rocks.com, R,squeaker@rocks.com, I\n" - "cdomain11.govReady,(blank),Federal-ExecutiveWorldWarICentennialCommissionmeoward@rocks.comR\n" - "cdomain1.gov,Ready,(blank),Federal - Executive,World War I Centennial Commission,,," - ", , , ,meoward@rocks.com,R,info@example.com,R,big_lebowski@dude.co,R," - "woofwardthethird@rocks.com,I\n" - "ddomain3.gov,On hold,(blank),Federal,Armed Forces Retirement Home,,,, , , ,,\n" - "zdomain12.gov,Ready,(blank),Interstate,meoward@rocks.com,R\n" - ) - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - - def test_export_data_managed_domains_to_csv(self): - """Test get counts for domains that have domain managers for two different dates, - get list of managed domains at end_date. - - An invited user, woofwardthethird, should also be pulled into this report.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_managed_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "MANAGED DOMAINS COUNTS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "MANAGED DOMAINS COUNTS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City," - "Special district,School district,Election office\n" - "3,2,1,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type,Domain manager 1,DM1 status,Domain manager 2,DM2 status," - "Domain manager 3,DM3 status,Domain manager 4,DM4 status\n" - "cdomain11.govFederal-Executivemeoward@rocks.com, R\n" - "cdomain1.gov,Federal - Executive,meoward@rocks.com,R,info@example.com,R," - "big_lebowski@dude.co,R,woofwardthethird@rocks.com,I\n" - "zdomain12.govInterstatemeoward@rocks.com,R\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_export_data_unmanaged_domains_to_csv(self): - """Test get counts for domains that do not have domain managers for two different dates, - get list of unmanaged domains at end_date.""" - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - export_data_unmanaged_domains_to_csv( - csv_file, self.start_date.strftime("%Y-%m-%d"), self.end_date.strftime("%Y-%m-%d") - ) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - - # We expect the READY domain names with the domain managers: Their counts, and listing at end_date. - expected_content = ( - "UNMANAGED DOMAINS AT START DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "0,0,0,0,0,0,0,0,0,0\n" - "\n" - "UNMANAGED DOMAINS AT END DATE\n" - "Total,Federal,Interstate,State or territory,Tribal,County,City,Special district," - "School district,Election office\n" - "1,1,0,0,0,0,0,0,0,0\n" - "\n" - "Domain name,Domain type\n" - "adomain10.gov,Federal\n" - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) - - def test_write_requests_body_with_date_filter_pulls_requests_in_range(self): - """Test that requests that are - 1. SUBMITTED and their submission_date are in range - are pulled when the growth report conditions are applied to export_requests_to_writed. - Test that requests are sorted by requested domain name. - """ - - with less_console_noise(): - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - # Define columns, sort fields, and filter condition - # We'll skip submission date because it's dynamic and therefore - # impossible to set in expected_content - columns = ["Domain request", "Domain type", "Federal type"] - sort_fields = [ - "requested_domain__name", - ] - filter_condition = { - "status": DomainRequest.DomainRequestStatus.SUBMITTED, - "submission_date__lte": self.end_date, - "submission_date__gte": self.start_date, - } - - additional_values = ["requested_domain__name"] - all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct() - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(all_requests, {}, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, columns, requests_dict) - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name - # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name expected_content = ( "Domain request,Domain type,Federal type\n" "city3.gov,Federal,Executive\n" @@ -705,70 +513,82 @@ class ExportDataTest(MockDb, MockEppLib): self.assertEqual(csv_content, expected_content) @less_console_noise_decorator - def test_full_domain_request_report(self): + def test_domain_request_data_full(self): """Tests the full domain request report.""" - - # Create a CSV file in memory - csv_file = StringIO() - writer = csv.writer(csv_file) - - # Call the report. Get existing fields from the report itself. - annotations = DomainRequestExport._full_domain_request_annotations() - additional_values = [ - "requested_domain__name", - "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", - "creator__first_name", - "creator__last_name", - "creator__email", - "investigator__email", + # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data + columns = [ + "Domain request", + # "Submitted at", + "Status", + "Domain type", + "Federal type", + "Federal agency", + "Organization name", + "Election office", + "City", + "State/territory", + "Region", + "Creator first name", + "Creator last name", + "Creator email", + "Creator approved domains count", + "Creator active requests count", + "Alternative domains", + "AO first name", + "AO last name", + "AO email", + "AO title/role", + "Request purpose", + "Request additional details", + "Other contacts", + "CISA regional representative", + "Current websites", + "Investigator", ] - requests = DomainRequest.objects.exclude(status=DomainRequest.DomainRequestStatus.STARTED) - annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(requests, annotations, additional_values) - requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False) - DomainRequestExport.write_csv_for_requests(writer, DomainRequestExport.all_columns, requests_dict) - - # Reset the CSV file's position to the beginning - csv_file.seek(0) - # Read the content into a variable - csv_content = csv_file.read() - print(csv_content) - self.maxDiff = None - expected_content = ( - # Header - "Domain request,Submitted at,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," - "AO title/role,Request purpose,Request additional details,Other contacts," - "CISA regional representative,Current websites,Investigator\n" - # Content - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city3.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1," - "cheeseville.gov | city1.gov | igorville.gov,Testy,Tester,testy@town.com,Chief Tester," - "Purpose of the site,CISA-first-name CISA-last-name | There is more,Meow Tester24 te2@town.com | " - "Testy1232 Tester24 te2@town.com | Testy Tester testy2@town.com,test@igorville.com," - "city.com | https://www.example2.com | https://www.example.com,\n" - "city4.gov,2024-04-02,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,\n" - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city6.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," - "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com," - ) - - # Normalize line endings and remove commas, - # spaces and leading/trailing whitespace - csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() - expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - - self.assertEqual(csv_content, expected_content) + with patch("registrar.utility.csv_export.DomainRequestDataFull.get_columns", return_value=columns): + # Create a CSV file in memory + csv_file = StringIO() + # Call the export functions + DomainRequestDataFull.export_data_to_csv(csv_file) + # Reset the CSV file's position to the beginning + csv_file.seek(0) + # Read the content into a variable + csv_content = csv_file.read() + print(csv_content) + expected_content = ( + # Header + "Domain request,Status,Domain type,Federal type," + "Federal agency,Organization name,Election office,City,State/territory," + "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," + "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," + "AO title/role,Request purpose,Request additional details,Other contacts," + "CISA regional representative,Current websites,Investigator\n" + # Content + "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," + "testy@town.com," + "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' + 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' + "CISA-last-name " + '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' + 'testy2@town.com"' + ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com" + ",cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " + "testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + ) + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + self.assertEqual(csv_content, expected_content) class HelperFunctions(MockDb): @@ -794,12 +614,12 @@ class HelperFunctions(MockDb): "domain__first_ready__lte": self.end_date, } # Test with distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) # Test without distinct - managed_domains_sliced_at_end_date = get_sliced_domains(filter_condition) + managed_domains_sliced_at_end_date = DomainExport.get_sliced_domains(filter_condition) expected_content = [3, 2, 1, 0, 0, 0, 0, 0, 0, 0] self.assertEqual(managed_domains_sliced_at_end_date, expected_content) @@ -811,6 +631,6 @@ class HelperFunctions(MockDb): "status": DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": self.end_date, } - submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition) + submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 6875f0e3d..f2786c2cd 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -15,12 +15,10 @@ from django.db.models import QuerySet, Value, CharField, Count, Q, F from django.db.models import Case, When, DateField from django.db.models import ManyToManyField from django.utils import timezone -from django.core.paginator import Paginator from django.db.models.functions import Concat, Coalesce from django.contrib.postgres.aggregates import StringAgg from registrar.models.utility.generic_helper import convert_queryset_to_dict from registrar.templatetags.custom_filters import get_region -from registrar.utility.enums import DefaultEmail from registrar.utility.constants import BranchChoices @@ -34,14 +32,17 @@ def write_header(writer, columns): """ writer.writerow(columns) + def get_default_start_date(): - # Default to a date that's prior to our first deployment + """Default to a date that's prior to our first deployment""" return timezone.make_aware(datetime(2023, 11, 1)) + def get_default_end_date(): - # Default to now() + """Default to now()""" return timezone.now() + def format_start_date(start_date): return timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() @@ -49,9 +50,11 @@ def format_start_date(start_date): def format_end_date(end_date): return timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + class BaseExport(ABC): """ A generic class for exporting data which returns a csv file for the given model. + Base class in an inheritance tree of 3. """ @classmethod @@ -69,14 +72,14 @@ class BaseExport(ABC): Returns the columns for CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields for the CSV export. Override in subclasses as needed. """ return [] - + @classmethod def get_additional_args(cls): """ @@ -84,63 +87,63 @@ class BaseExport(ABC): Override in subclasses to provide specific arguments. """ return {} - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ return [] - + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ return [] - + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ return Q() - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ Get a Q object of filter conditions to filter when building queryset. """ return Q() - + @classmethod def get_computed_fields(cls): """ Get a dict of computed fields. """ return {} - + @classmethod def get_annotations_for_sort(cls): """ Get a dict of annotations to make available for order_by clause. """ return {} - + @classmethod def get_related_table_fields(cls): """ Get a list of fields from related tables. """ return [] - + @classmethod def update_queryset(cls, queryset, **kwargs): """ Returns an updated queryset. Override in subclass to update queryset. """ return queryset - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -187,7 +190,7 @@ class BaseExport(ABC): queryset = initial_queryset.annotate(**computed_fields).values(*model_fields, *related_table_fields) return cls.update_queryset(queryset, **kwargs) - + @classmethod def export_data_to_csv(cls, csv_file, start_date=None, end_date=None): """ @@ -207,7 +210,8 @@ class BaseExport(ABC): related_table_fields = cls.get_related_table_fields() model_queryset = ( - cls.model().objects.select_related(*select_related) + cls.model() + .objects.select_related(*select_related) .prefetch_related(*prefetch_related) .filter(filter_conditions) .exclude(exclusions) @@ -217,7 +221,9 @@ class BaseExport(ABC): ) # Convert the queryset to a dictionary (including annotated fields) - annotated_queryset = cls.annotate_and_retrieve_fields(model_queryset, computed_fields, related_table_fields, **kwargs) + annotated_queryset = cls.annotate_and_retrieve_fields( + model_queryset, computed_fields, related_table_fields, **kwargs + ) models_dict = convert_queryset_to_dict(annotated_queryset, is_model=False) # Write to csv file before the write_csv @@ -259,10 +265,11 @@ class BaseExport(ABC): """ pass - + class DomainExport(BaseExport): """ A collection of functions which return csv files regarding the Domain model. + Second class in an inheritance tree of 3. """ @classmethod @@ -279,9 +286,9 @@ class DomainExport(BaseExport): based on public_contacts, domain_invitations and user_domain_roles passed through kwargs. """ - public_contacts = kwargs.get('public_contacts', {}) - domain_invitations = kwargs.get('domain_invitations', {}) - user_domain_roles = kwargs.get('user_domain_roles', {}) + public_contacts = kwargs.get("public_contacts", {}) + domain_invitations = kwargs.get("domain_invitations", {}) + user_domain_roles = kwargs.get("user_domain_roles", {}) annotated_domain_infos = [] @@ -296,16 +303,18 @@ class DomainExport(BaseExport): # Annotate with security_contact from public_contacts for domain_info in queryset: - domain_info['security_contact_email'] = public_contacts.get(domain_info.get('domain__security_contact_registry_id')) - domain_info['invited_users'] = ', '.join(invited_users_dict.get(domain_info.get('domain__name'), [])) - domain_info['managers'] = ', '.join(managers_dict.get(domain_info.get('domain__name'), [])) + domain_info["security_contact_email"] = public_contacts.get( + domain_info.get("domain__security_contact_registry_id") + ) + domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) + domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) annotated_domain_infos.append(domain_info) if annotated_domain_infos: return annotated_domain_infos - + return queryset - + # ============================================================= # # Helper functions for django ORM queries. # # We are using these rather than pure python for speed reasons. # @@ -316,15 +325,15 @@ class DomainExport(BaseExport): """ Fetch all PublicContact entries and return a mapping of registry_id to email. """ - public_contacts = PublicContact.objects.values_list('registry_id', 'email') + public_contacts = PublicContact.objects.values_list("registry_id", "email") return {registry_id: email for registry_id, email in public_contacts} - + @classmethod def get_all_domain_invitations(cls): """ Fetch all DomainInvitation entries and return a mapping of domain to email. """ - domain_invitations = DomainInvitation.objects.filter(status="invited").values_list('domain__name', 'email') + domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") return list(domain_invitations) @classmethod @@ -332,7 +341,7 @@ class DomainExport(BaseExport): """ Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ - user_domain_roles = UserDomainRole.objects.select_related('user').values_list('domain__name', 'user__email') + user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") return list(user_domain_roles) @classmethod @@ -360,19 +369,9 @@ class DomainExport(BaseExport): if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: domain_type = f"{human_readable_domain_org_type} - {human_readable_domain_federal_type}" - if model.get("domain__name") == "18f.gov": - print(f'domain_type {domain_type}') - print(f'federal_agency {model.get("federal_agency")}') - print(f'city {model.get("city")}') - - print(f'agency {model.get("agency")}') - - print(f'federal_agency__agency {model.get("federal_agency__agency")}') - # create a dictionary of fields which can be included in output. # "extra_fields" are precomputed fields (generated in the DB or parsed). FIELDS = { - "Domain name": model.get("domain__name"), "Status": human_readable_status, "First ready on": first_ready_on, @@ -434,6 +433,10 @@ class DomainExport(BaseExport): class DomainDataType(DomainExport): + """ + Shows security contacts, domain managers, ao + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -456,7 +459,7 @@ class DomainDataType(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ @@ -488,29 +491,24 @@ class DomainDataType(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'public_contacts': public_contacts, - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "public_contacts": public_contacts, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain", - "authorizing_official" - ] - + return ["domain", "authorizing_official"] + @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] + return ["permissions"] @classmethod def get_computed_fields(cls, delimiter=", "): @@ -525,7 +523,7 @@ class DomainDataType(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -542,9 +540,13 @@ class DomainDataType(DomainExport): "authorizing_official__email", "federal_agency__agency", ] - + class DomainDataFull(DomainExport): + """ + Shows security contacts, filtered by state + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -560,7 +562,7 @@ class DomainDataFull(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -586,17 +588,15 @@ class DomainDataFull(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -604,13 +604,13 @@ class DomainDataFull(DomainExport): Get a Q object of filter conditions to filter when building queryset. """ return Q( - domain__state__in = [ + domain__state__in=[ Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -624,7 +624,7 @@ class DomainDataFull(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -638,6 +638,10 @@ class DomainDataFull(DomainExport): class DomainDataFederal(DomainExport): + """ + Shows security contacts, filtered by state and org type + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -653,7 +657,7 @@ class DomainDataFederal(DomainExport): "State", "Security contact email", ] - + @classmethod def get_sort_fields(cls): """ @@ -679,17 +683,15 @@ class DomainDataFederal(DomainExport): public_contacts = cls.get_all_security_emails() return { - 'public_contacts': public_contacts, + "public_contacts": public_contacts, } - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -702,9 +704,9 @@ class DomainDataFederal(DomainExport): Domain.State.READY, Domain.State.DNS_NEEDED, Domain.State.ON_HOLD, - ] + ], ) - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -718,7 +720,7 @@ class DomainDataFederal(DomainExport): output_field=CharField(), ), } - + @classmethod def get_related_table_fields(cls): """ @@ -732,6 +734,10 @@ class DomainDataFederal(DomainExport): class DomainGrowth(DomainExport): + """ + Shows ready and deleted domains within a date range, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -751,7 +757,7 @@ class DomainGrowth(DomainExport): "First ready", "Deleted", ] - + @classmethod def get_annotations_for_sort(cls, delimiter=", "): """ @@ -760,10 +766,10 @@ class DomainGrowth(DomainExport): today = timezone.now().date() return { "custom_sort": Case( - When(domain__state=Domain.State.READY, then='domain__first_ready'), - When(domain__state=Domain.State.DELETED, then='domain__deleted'), + When(domain__state=Domain.State.READY, then="domain__first_ready"), + When(domain__state=Domain.State.DELETED, then="domain__deleted"), default=Value(today), # Default value if no conditions match - output_field=DateField() + output_field=DateField(), ) } @@ -773,19 +779,17 @@ class DomainGrowth(DomainExport): Returns the sort fields. """ return [ - '-domain__state', - 'custom_sort', - 'domain__name', + "-domain__state", + "custom_sort", + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): @@ -795,15 +799,13 @@ class DomainGrowth(DomainExport): filter_ready = Q( domain__state__in=[Domain.State.READY], domain__first_ready__gte=start_date, - domain__first_ready__lte=end_date + domain__first_ready__lte=end_date, ) filter_deleted = Q( - domain__state__in=[Domain.State.DELETED], - domain__deleted__gte=start_date, - domain__deleted__lte=end_date + domain__state__in=[Domain.State.DELETED], domain__deleted__gte=start_date, domain__deleted__lte=end_date ) return filter_ready | filter_deleted - + @classmethod def get_related_table_fields(cls): """ @@ -821,6 +823,10 @@ class DomainGrowth(DomainExport): class DomainManaged(DomainExport): + """ + Shows managed domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -833,34 +839,30 @@ class DomainManaged(DomainExport): "Domain managers", "Invited domain managers", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -871,7 +873,6 @@ class DomainManaged(DomainExport): domain__permissions__isnull=False, domain__first_ready__lte=end_date_formatted, ) - @classmethod def get_additional_args(cls): @@ -889,10 +890,10 @@ class DomainManaged(DomainExport): user_domain_roles = cls.get_all_user_domain_roles() return { - 'domain_invitations': domain_invitations, - 'user_domain_roles': user_domain_roles, + "domain_invitations": domain_invitations, + "user_domain_roles": user_domain_roles, } - + @classmethod def get_related_table_fields(cls): """ @@ -901,7 +902,7 @@ class DomainManaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ @@ -959,6 +960,10 @@ class DomainManaged(DomainExport): class DomainUnmanaged(DomainExport): + """ + Shows unmanaged domains by an end date, sorted + Inherits from BaseExport -> DomainExport + """ @classmethod def get_columns(cls): @@ -969,34 +974,30 @@ class DomainUnmanaged(DomainExport): "Domain name", "Domain type", ] - + @classmethod def get_sort_fields(cls): """ Returns the sort fields. """ return [ - 'domain__name', + "domain__name", ] - + @classmethod def get_select_related(cls): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "domain" - ] + return ["domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "permissions" - ] - + return ["permissions"] + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1007,7 +1008,7 @@ class DomainUnmanaged(DomainExport): domain__permissions__isnull=True, domain__first_ready__lte=end_date_formatted, ) - + @classmethod def get_related_table_fields(cls): """ @@ -1016,12 +1017,12 @@ class DomainUnmanaged(DomainExport): return [ "domain__name", ] - + @classmethod def write_csv_before(cls, csv_writer, start_date=None, end_date=None): """ Write to csv file before the write_csv method. - + """ start_date_formatted = format_start_date(start_date) end_date_formatted = format_end_date(end_date) @@ -1075,6 +1076,10 @@ class DomainUnmanaged(DomainExport): class DomainRequestExport(BaseExport): + """ + A collection of functions which return csv files regarding the DomainRequest model. + Second class in an inheritance tree of 3. + """ @classmethod def model(cls): @@ -1197,6 +1202,10 @@ class DomainRequestExport(BaseExport): class DomainRequestGrowth(DomainRequestExport): + """ + Shows submitted requests within a date range, sorted + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1218,7 +1227,7 @@ class DomainRequestGrowth(DomainRequestExport): return [ "requested_domain__name", ] - + @classmethod def get_filter_conditions(cls, start_date=None, end_date=None): """ @@ -1238,12 +1247,14 @@ class DomainRequestGrowth(DomainRequestExport): """ Get a list of fields from related tables. """ - return [ - "requested_domain__name" - ] - + return ["requested_domain__name"] + class DomainRequestDataFull(DomainRequestExport): + """ + Shows all but STARTED requests + Inherits from BaseExport -> DomainRequestExport + """ @classmethod def get_columns(cls): @@ -1285,34 +1296,22 @@ class DomainRequestDataFull(DomainRequestExport): """ Get a list of tables to pass to select_related when building queryset. """ - return [ - "creator", - "authorizing_official", - "federal_agency", - "investigator", - "requested_domain" - ] + return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"] @classmethod def get_prefetch_related(cls): """ Get a list of tables to pass to prefetch_related when building queryset. """ - return [ - "current_websites", - "other_contacts", - "alternative_domains" - ] - + return ["current_websites", "other_contacts", "alternative_domains"] + @classmethod def get_exclusions(cls): """ Get a Q object of exclusion conditions to use when building queryset. """ - return Q( - status__in=[DomainRequest.DomainRequestStatus.STARTED] - ) - + return Q(status__in=[DomainRequest.DomainRequestStatus.STARTED]) + @classmethod def get_sort_fields(cls): """ @@ -1322,7 +1321,7 @@ class DomainRequestDataFull(DomainRequestExport): "status", "requested_domain__name", ] - + @classmethod def get_computed_fields(cls, delimiter=", "): """ @@ -1346,7 +1345,7 @@ class DomainRequestDataFull(DomainRequestExport): distinct=True, ), } - + @classmethod def get_related_table_fields(cls): """ @@ -1364,7 +1363,6 @@ class DomainRequestDataFull(DomainRequestExport): "creator__email", "investigator__email", ] - # ============================================================= # # Helper functions for django ORM queries. # diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py index 63e8492b5..4d015ab37 100644 --- a/src/registrar/views/admin_views.py +++ b/src/registrar/views/admin_views.py @@ -49,7 +49,9 @@ class AnalyticsView(View): "domain__permissions__isnull": False, "domain__first_ready__lte": end_date_formatted, } - managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_start_date) + managed_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_managed_domains_start_date + ) managed_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_managed_domains_end_date) filter_unmanaged_domains_start_date = { @@ -60,8 +62,12 @@ class AnalyticsView(View): "domain__permissions__isnull": True, "domain__first_ready__lte": end_date_formatted, } - unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_start_date) - unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_unmanaged_domains_end_date) + unmanaged_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_start_date + ) + unmanaged_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains( + filter_unmanaged_domains_end_date + ) filter_ready_domains_start_date = { "domain__state__in": [models.Domain.State.READY], @@ -82,7 +88,9 @@ class AnalyticsView(View): "domain__state__in": [models.Domain.State.DELETED], "domain__deleted__lte": end_date_formatted, } - deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_start_date) + deleted_domains_sliced_at_start_date = csv_export.DomainExport.get_sliced_domains( + filter_deleted_domains_start_date + ) deleted_domains_sliced_at_end_date = csv_export.DomainExport.get_sliced_domains(filter_deleted_domains_end_date) filter_requests_start_date = { @@ -102,8 +110,12 @@ class AnalyticsView(View): "status": models.DomainRequest.DomainRequestStatus.SUBMITTED, "submission_date__lte": end_date_formatted, } - submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_start_date) - submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests(filter_submitted_requests_end_date) + submitted_requests_sliced_at_start_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_start_date + ) + submitted_requests_sliced_at_end_date = csv_export.DomainRequestExport.get_sliced_requests( + filter_submitted_requests_end_date + ) context = dict( # Generate a dictionary of context variables that are common across all admin templates From bdac7ed04f0ecdd31700a5f93f02c9031f2f66ff Mon Sep 17 00:00:00 2001 From: CocoByte Date: Fri, 28 Jun 2024 16:26:11 -0600 Subject: [PATCH 17/55] fix migrations --- .../migrations/0108_create_groups_v14.py | 37 ------------------- ...niorofficial_portfolio_senior_official.py} | 4 +- 2 files changed, 2 insertions(+), 39 deletions(-) delete mode 100644 src/registrar/migrations/0108_create_groups_v14.py rename src/registrar/migrations/{0107_seniorofficial_portfolio_senior_official.py => 0108_seniorofficial_portfolio_senior_official.py} (92%) diff --git a/src/registrar/migrations/0108_create_groups_v14.py b/src/registrar/migrations/0108_create_groups_v14.py deleted file mode 100644 index e171156fb..000000000 --- a/src/registrar/migrations/0108_create_groups_v14.py +++ /dev/null @@ -1,37 +0,0 @@ -# This migration creates the create_full_access_group and create_cisa_analyst_group groups -# It is dependent on 0079 (which populates federal agencies) -# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS -# in the user_group model then: -# [NOT RECOMMENDED] -# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions -# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups -# step 3: fake run the latest migration in the migrations list -# [RECOMMENDED] -# Alternatively: -# step 1: duplicate the migration that loads data -# step 2: docker-compose exec app ./manage.py migrate - -from django.db import migrations -from registrar.models import UserGroup -from typing import Any - - -# For linting: RunPython expects a function reference, -# so let's give it one -def create_groups(apps, schema_editor) -> Any: - UserGroup.create_cisa_analyst_group(apps, schema_editor) - UserGroup.create_full_access_group(apps, schema_editor) - - -class Migration(migrations.Migration): - dependencies = [ - ("registrar", "0107_seniorofficial_portfolio_senior_official"), - ] - - operations = [ - migrations.RunPython( - create_groups, - reverse_code=migrations.RunPython.noop, - atomic=True, - ), - ] diff --git a/src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py similarity index 92% rename from src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py rename to src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py index 226822990..e00330159 100644 --- a/src/registrar/migrations/0107_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-06-25 20:42 +# Generated by Django 4.2.10 on 2024-06-28 18:38 from django.db import migrations, models import django.db.models.deletion @@ -8,7 +8,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0106_create_groups_v14"), + ("registrar", "0107_domainrequest_action_needed_reason_email"), ] operations = [ From ec6fd51ca1fe9975a4cce819f925eb4c94de3c94 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 2 Jul 2024 09:00:44 -0600 Subject: [PATCH 18/55] update migrations --- ...niorofficial_portfolio_senior_official.py} | 4 +- .../migrations/0110_create_groups_v15.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) rename src/registrar/migrations/{0108_seniorofficial_portfolio_senior_official.py => 0109_seniorofficial_portfolio_senior_official.py} (92%) create mode 100644 src/registrar/migrations/0110_create_groups_v15.py diff --git a/src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py similarity index 92% rename from src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py rename to src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py index e00330159..9e4d0b7b1 100644 --- a/src/registrar/migrations/0108_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-06-28 18:38 +# Generated by Django 4.2.10 on 2024-07-02 14:59 from django.db import migrations, models import django.db.models.deletion @@ -8,7 +8,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0107_domainrequest_action_needed_reason_email"), + ("registrar", "0108_domaininformation_authorizing_official_and_more"), ] operations = [ diff --git a/src/registrar/migrations/0110_create_groups_v15.py b/src/registrar/migrations/0110_create_groups_v15.py new file mode 100644 index 000000000..bf7c7f325 --- /dev/null +++ b/src/registrar/migrations/0110_create_groups_v15.py @@ -0,0 +1,37 @@ +# This migration creates the create_full_access_group and create_cisa_analyst_group groups +# It is dependent on 0079 (which populates federal agencies) +# If permissions on the groups need changing, edit CISA_ANALYST_GROUP_PERMISSIONS +# in the user_group model then: +# [NOT RECOMMENDED] +# step 1: docker-compose exec app ./manage.py migrate --fake registrar 0035_contenttypes_permissions +# step 2: docker-compose exec app ./manage.py migrate registrar 0036_create_groups +# step 3: fake run the latest migration in the migrations list +# [RECOMMENDED] +# Alternatively: +# step 1: duplicate the migration that loads data +# step 2: docker-compose exec app ./manage.py migrate + +from django.db import migrations +from registrar.models import UserGroup +from typing import Any + + +# For linting: RunPython expects a function reference, +# so let's give it one +def create_groups(apps, schema_editor) -> Any: + UserGroup.create_cisa_analyst_group(apps, schema_editor) + UserGroup.create_full_access_group(apps, schema_editor) + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0109_seniorofficial_portfolio_senior_official"), + ] + + operations = [ + migrations.RunPython( + create_groups, + reverse_code=migrations.RunPython.noop, + atomic=True, + ), + ] From 6361369953c1a4cf2b0d92ed3eb18876a943fae1 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 15:10:49 -0400 Subject: [PATCH 19/55] removed user-contact connection --- docs/developer/README.md | 43 -------- src/registrar/apps.py | 9 -- src/registrar/forms/__init__.py | 2 +- src/registrar/forms/domain.py | 59 ++++++++++- src/registrar/forms/user_profile.py | 4 +- .../management/commands/import_tables.py | 7 -- src/registrar/models/contact.py | 7 -- src/registrar/models/user.py | 23 ++-- src/registrar/signals.py | 59 ----------- .../admin/includes/contact_detail_list.html | 27 ++--- src/registrar/templates/domain_detail.html | 2 +- .../includes/finish_profile_form.html | 4 +- .../includes/profile_information.html | 6 +- src/registrar/tests/common.py | 4 + src/registrar/tests/test_admin.py | 47 +++----- src/registrar/tests/test_models.py | 98 +---------------- src/registrar/tests/test_signals.py | 100 ------------------ src/registrar/tests/test_views.py | 14 ++- src/registrar/tests/test_views_domain.py | 4 +- src/registrar/views/domain.py | 6 +- src/registrar/views/user_profile.py | 16 ++- .../views/utility/permission_views.py | 6 +- 22 files changed, 129 insertions(+), 418 deletions(-) delete mode 100644 src/registrar/signals.py delete mode 100644 src/registrar/tests/test_signals.py diff --git a/docs/developer/README.md b/docs/developer/README.md index 72f6b9f20..f63f01938 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -353,49 +353,6 @@ cf env getgov-{app name} Then, copy the variables under the section labled `s3`. -## Signals -The application uses [Django signals](https://docs.djangoproject.com/en/5.0/topics/signals/). In particular, it uses a subset of prebuilt signals called [model signals](https://docs.djangoproject.com/en/5.0/ref/signals/#module-django.db.models.signals). - -Per Django, signals "[...allow certain senders to notify a set of receivers that some action has taken place.](https://docs.djangoproject.com/en/5.0/topics/signals/#module-django.dispatch)" - -In other words, signals are a mechanism that allows different parts of an application to communicate with each other by sending and receiving notifications when events occur. When an event occurs (such as creating, updating, or deleting a record), signals can automatically trigger specific actions in response. This allows different parts of an application to stay synchronized without tightly coupling the component. - -### Rules of use -When using signals, try to adhere to these guidelines: -1. Don't use signals when you can use another method, such as an override of `save()` or `__init__`. -2. Document its usage in this readme (or another centralized location), as well as briefly on the underlying class it is associated with. For instance, since the `handle_profile` directly affects the class `Contact`, the class description notes this and links to [signals.py](../../src/registrar/signals.py). -3. Where possible, avoid chaining signals together (i.e. a signal that calls a signal). If this has to be done, clearly document the flow. -4. Minimize logic complexity within the signal as much as possible. - -### When should you use signals? -Generally, you would use signals when you want an event to be synchronized across multiple areas of code at once (such as with two models or more models at once) in a way that would otherwise be difficult to achieve by overriding functions. - -However, in most scenarios, if you can get away with avoiding signals - you should. The reasoning for this is that [signals give the appearance of loose coupling, but they can quickly lead to code that is hard to understand, adjust and debug](https://docs.djangoproject.com/en/5.0/topics/signals/#module-django.dispatch). - -Consider using signals when: -1. Synchronizing events across multiple models or areas of code. -2. Performing logic before or after saving a model to the database (when otherwise difficult through `save()`). -3. Encountering an import loop when overriding functions such as `save()`. -4. You are otherwise unable to achieve the intended behavior by overrides or other means. -5. (Rare) Offloading tasks when multi-threading. - -For the vast majority of use cases, the [pre_save](https://docs.djangoproject.com/en/5.0/ref/signals/#pre-save) and [post_save](https://docs.djangoproject.com/en/5.0/ref/signals/#post-save) signals are sufficient in terms of model-to-model management. - -### Where should you use them? -This project compiles signals in a unified location to maintain readability. If you are adding a signal or otherwise utilizing one, you should always define them in [signals.py](../../src/registrar/signals.py). Except under rare circumstances, this should be adhered to for the reasons mentioned above. - -### How are we currently using signals? -At the time of writing, we currently only use signals for the Contact and User objects when synchronizing data returned from Login.gov. This is because the `Contact` object holds information that the user specified in our system, whereas the `User` object holds information that was specified in Login.gov. - -To keep our signal usage coherent and well-documented, add to this document when a new function is added for ease of reference and use. - -#### handle_profile -This function is triggered by the post_save event on the User model, designed to manage the synchronization between User and Contact entities. It operates under the following conditions: - -1. For New Users: Upon the creation of a new user, it checks for an existing `Contact` by email. If no matching contact is found, it creates a new Contact using the user's details from Login.gov. If a matching contact is found, it associates this contact with the user. In cases where multiple contacts with the same email exist, it logs a warning and associates the first contact found. - -2. For Existing Users: For users logging in subsequent times, the function ensures that any updates from Login.gov are applied to the associated User record. However, it does not alter any existing Contact records. - ## Disable email sending (toggling the disable_email_sending flag) 1. On the app, navigate to `\admin`. 2. Under models, click `Waffle flags`. diff --git a/src/registrar/apps.py b/src/registrar/apps.py index fcb5c17fd..b5952208b 100644 --- a/src/registrar/apps.py +++ b/src/registrar/apps.py @@ -5,12 +5,3 @@ class RegistrarConfig(AppConfig): """Configure signal handling for our registrar Django application.""" name = "registrar" - - def ready(self): - """Runs when all Django applications have been loaded. - - We use it here to load signals that connect related models. - """ - # noqa here because we are importing something to make the signals - # get registered, but not using what we import - from . import signals # noqa diff --git a/src/registrar/forms/__init__.py b/src/registrar/forms/__init__.py index be3b634f6..8f4f8ea14 100644 --- a/src/registrar/forms/__init__.py +++ b/src/registrar/forms/__init__.py @@ -4,7 +4,7 @@ from .domain import ( NameserverFormset, DomainSecurityEmailForm, DomainOrgNameAddressForm, - ContactForm, + UserForm, AuthorizingOfficialContactForm, DomainDnssecForm, DomainDsdataFormset, diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 0e9fbb693..6f6c9077e 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -16,7 +16,7 @@ from registrar.utility.errors import ( SecurityEmailErrorCodes, ) -from ..models import Contact, DomainInformation, Domain +from ..models import Contact, DomainInformation, Domain, User from .common import ( ALGORITHM_CHOICES, DIGEST_TYPE_CHOICES, @@ -203,6 +203,63 @@ NameserverFormset = formset_factory( ) +class UserForm(forms.ModelForm): + """Form for updating contacts.""" + + email = forms.EmailField(max_length=None) + + class Meta: + model = User + fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] + widgets = { + "first_name": forms.TextInput, + "middle_name": forms.TextInput, + "last_name": forms.TextInput, + "title": forms.TextInput, + "email": forms.EmailInput, + "phone": RegionalPhoneNumberWidget, + } + + # the database fields have blank=True so ModelForm doesn't create + # required fields by default. Use this list in __init__ to mark each + # of these fields as required + required = ["first_name", "last_name", "title", "email", "phone"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # take off maxlength attribute for the phone number field + # which interferes with out input_with_errors template tag + self.fields["phone"].widget.attrs.pop("maxlength", None) + + # Define a custom validator for the email field with a custom error message + email_max_length_validator = MaxLengthValidator(320, message="Response must be less than 320 characters.") + self.fields["email"].validators.append(email_max_length_validator) + + for field_name in self.required: + self.fields[field_name].required = True + + # Set custom form label + self.fields["middle_name"].label = "Middle name (optional)" + + # Set custom error messages + self.fields["first_name"].error_messages = {"required": "Enter your first name / given name."} + self.fields["last_name"].error_messages = {"required": "Enter your last name / family name."} + self.fields["title"].error_messages = { + "required": "Enter your title or role in your organization (e.g., Chief Information Officer)" + } + self.fields["email"].error_messages = { + "required": "Enter your email address in the required format, like name@example.com." + } + self.fields["phone"].error_messages["required"] = "Enter your phone number." + self.domainInfo = None + + def set_domain_info(self, domainInfo): + """Set the domain information for the form. + The form instance is associated with the contact itself. In order to access the associated + domain information object, this needs to be set in the form by the view.""" + self.domainInfo = domainInfo + + class ContactForm(forms.ModelForm): """Form for updating contacts.""" diff --git a/src/registrar/forms/user_profile.py b/src/registrar/forms/user_profile.py index 682e1a5df..bfdcd0da8 100644 --- a/src/registrar/forms/user_profile.py +++ b/src/registrar/forms/user_profile.py @@ -1,6 +1,6 @@ from django import forms -from registrar.models.contact import Contact +from registrar.models.user import User from django.core.validators import MaxLengthValidator from phonenumber_field.widgets import RegionalPhoneNumberWidget @@ -13,7 +13,7 @@ class UserProfileForm(forms.ModelForm): redirect = forms.CharField(widget=forms.HiddenInput(), required=False) class Meta: - model = Contact + model = User fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] widgets = { "first_name": forms.TextInput, diff --git a/src/registrar/management/commands/import_tables.py b/src/registrar/management/commands/import_tables.py index cb78e13bd..62562e7f7 100644 --- a/src/registrar/management/commands/import_tables.py +++ b/src/registrar/management/commands/import_tables.py @@ -65,13 +65,6 @@ class Command(BaseCommand): resourcename = f"{table_name}Resource" - # if table_name is Contact, clean the table first - # User table is loaded before Contact, and signals create - # rows in Contact table which break the import, so need - # to be cleaned again before running import on Contact table - if table_name == "Contact": - self.clean_table(table_name) - # Define the directory and the pattern for csv filenames tmp_dir = "tmp" pattern = f"{table_name}_" diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index f94938dd1..f7bae3491 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -8,13 +8,6 @@ from phonenumber_field.modelfields import PhoneNumberField # type: ignore class Contact(TimeStampedModel): """ Contact information follows a similar pattern for each contact. - - This model uses signals [as defined in [signals.py](../../src/registrar/signals.py)]. - When a new user is created through Login.gov, a contact object will be created and - associated on the `user` field. - - If the `user` object already exists, the underlying user object - will be updated if any updates are made to it through Login.gov. """ class Meta: diff --git a/src/registrar/models/user.py b/src/registrar/models/user.py index bb0276607..87b7799d3 100644 --- a/src/registrar/models/user.py +++ b/src/registrar/models/user.py @@ -23,10 +23,6 @@ class User(AbstractUser): A custom user model that performs identically to the default user model but can be customized later. - This model uses signals [as defined in [signals.py](../../src/registrar/signals.py)]. - When a new user is created through Login.gov, a contact object will be created and - associated on the contacts `user` field. - If the `user` object already exists, said user object will be updated if any updates are made to it through Login.gov. """ @@ -113,15 +109,11 @@ class User(AbstractUser): Tracks if the user finished their profile setup or not. This is so we can globally enforce that new users provide additional account information before proceeding. """ - - # Change this to self once the user and contact objects are merged. - # For now, since they are linked, lets test on the underlying contact object. - user_info = self.contact # noqa user_values = [ - user_info.first_name, - user_info.last_name, - user_info.title, - user_info.phone, + self.first_name, + self.last_name, + self.title, + self.phone, ] return None not in user_values @@ -169,8 +161,13 @@ class User(AbstractUser): """Return count of ineligible requests""" return self.domain_requests_created.filter(status=DomainRequest.DomainRequestStatus.INELIGIBLE).count() + def get_formatted_name(self): + """Returns the contact's name in Western order.""" + names = [n for n in [self.first_name, self.middle_name, self.last_name] if n] + return " ".join(names) if names else "Unknown" + def has_contact_info(self): - return bool(self.contact.title or self.contact.email or self.contact.phone) + return bool(self.title or self.email or self.phone) @classmethod def needs_identity_verification(cls, email, uuid): diff --git a/src/registrar/signals.py b/src/registrar/signals.py deleted file mode 100644 index bc0480b2a..000000000 --- a/src/registrar/signals.py +++ /dev/null @@ -1,59 +0,0 @@ -import logging - -from django.db.models.signals import post_save -from django.dispatch import receiver - -from .models import User, Contact - - -logger = logging.getLogger(__name__) - - -@receiver(post_save, sender=User) -def handle_profile(sender, instance, **kwargs): - """Method for when a User is saved. - - A first time registrant may have been invited, so we'll search for a matching - Contact record, by email address, and associate them, if possible. - - A first time registrant may not have a matching Contact, so we'll create one, - copying the contact values we received from Login.gov in order to initialize it. - - During subsequent login, a User record may be updated with new data from Login.gov, - but in no case will we update contact values on an existing Contact record. - """ - - first_name = getattr(instance, "first_name", "") - middle_name = getattr(instance, "middle_name", "") - last_name = getattr(instance, "last_name", "") - email = getattr(instance, "email", "") - phone = getattr(instance, "phone", "") - title = getattr(instance, "title", "") - - is_new_user = kwargs.get("created", False) - - if is_new_user: - contacts = Contact.objects.filter(email=email) - else: - contacts = Contact.objects.filter(user=instance) - - if len(contacts) == 0: # no matching contact - Contact.objects.create( - user=instance, - first_name=first_name, - middle_name=middle_name, - last_name=last_name, - email=email, - phone=phone, - title=title, - ) - - if len(contacts) >= 1 and is_new_user: # a matching contact - contacts[0].user = instance - contacts[0].save() - - if len(contacts) > 1: # multiple matches - logger.warning( - "There are multiple Contacts with the same email address." - f" Picking #{contacts[0].id} for User #{instance.id}." - ) diff --git a/src/registrar/templates/django/admin/includes/contact_detail_list.html b/src/registrar/templates/django/admin/includes/contact_detail_list.html index 3b49e62a4..2ee490d76 100644 --- a/src/registrar/templates/django/admin/includes/contact_detail_list.html +++ b/src/registrar/templates/django/admin/includes/contact_detail_list.html @@ -12,37 +12,24 @@ {% if user.has_contact_info %} {# Title #} - {% if user.title or user.contact.title %} - {% if user.contact.title %} - {{ user.contact.title }} - {% else %} - {{ user.title }} - {% endif %} + {% if user.title %} + {{ user.title }}
{% else %} None
{% endif %} {# Email #} - {% if user.email or user.contact.email %} - {% if user.contact.email %} - {{ user.contact.email }} - {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %} - {% else %} - {{ user.email }} - {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %} - {% endif %} + {% if user.email %} + {{ user.email }} + {% include "admin/input_with_clipboard.html" with field=user invisible_input_field=True %}
{% else %} None
{% endif %} {# Phone #} - {% if user.phone or user.contact.phone %} - {% if user.contact.phone %} - {{ user.contact.phone }} - {% else %} - {{ user.phone }} - {% endif %} + {% if user.phone %} + {{ user.phone }}
{% else %} None
diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index 815df4942..d201831cc 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -62,7 +62,7 @@ {# Conditionally display profile #} {% if not has_profile_feature_flag %} {% url 'domain-your-contact-information' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Your contact information' value=request.user.contact contact='true' edit_link=url editable=domain.is_editable %} + {% include "includes/summary_item.html" with title='Your contact information' value=request.user contact='true' edit_link=url editable=domain.is_editable %} {% endif %} {% url 'domain-security-email' pk=domain.id as url %} diff --git a/src/registrar/templates/includes/finish_profile_form.html b/src/registrar/templates/includes/finish_profile_form.html index d43ff812c..88f7a73af 100644 --- a/src/registrar/templates/includes/finish_profile_form.html +++ b/src/registrar/templates/includes/finish_profile_form.html @@ -77,11 +77,11 @@
- {% if user_finished_setup and going_to_specific_page %} - {% endif %} diff --git a/src/registrar/templates/includes/profile_information.html b/src/registrar/templates/includes/profile_information.html index 2922fd3f7..3e7c827f1 100644 --- a/src/registrar/templates/includes/profile_information.html +++ b/src/registrar/templates/includes/profile_information.html @@ -13,10 +13,10 @@
    -
  • Full name: {{ user.contact.get_formatted_name }}
    • +
  • Full name: {{ user.get_formatted_name }}
  • Organization email: {{ user.email }}
    • -
  • Title or role in your organization: {{ user.contact.title }}
    • -
  • Phone: {{ user.contact.phone.as_national }}
    • +
  • Title or role in your organization: {{ user.title }}
    • +
  • Phone: {{ user.phone.as_national }}
diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 017964299..a8bdfec29 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -810,6 +810,8 @@ def create_superuser(): user = User.objects.create_user( username="superuser", email="admin@example.com", + first_name="first", + last_name="last", is_staff=True, password=p, ) @@ -826,6 +828,8 @@ def create_user(): user = User.objects.create_user( username="staffuser", email="staff@example.com", + first_name="first", + last_name="last", is_staff=True, password=p, ) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 07e97d159..ff68e5a7c 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -242,15 +242,11 @@ class TestDomainAdmin(MockEppLib, WebTest): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) domain_request.approve() @@ -2067,15 +2063,11 @@ class TestDomainRequestAdmin(MockEppLib): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) @@ -2092,11 +2084,11 @@ class TestDomainRequestAdmin(MockEppLib): # == Check for the creator == # - # Check for the right title, email, and phone number in the response. + # Check for the right title and phone number in the response. + # Email will appear more than once expected_creator_fields = [ # Field, expected value ("title", "Treat inspector"), - ("email", "meoward.jones@igorville.gov"), ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) @@ -3103,15 +3095,11 @@ class TestDomainInformationAdmin(TestCase): username="MrMeoward", first_name="Meoward", last_name="Jones", + email="meoward.jones@igorville.gov", + phone="(555) 123 12345", + title="Treat inspector", ) - # Due to the relation between User <==> Contact, - # the underlying contact has to be modified this way. - _creator.contact.email = "meoward.jones@igorville.gov" - _creator.contact.phone = "(555) 123 12345" - _creator.contact.title = "Treat inspector" - _creator.contact.save() - # Create a fake domain request domain_request = completed_domain_request(status=DomainRequest.DomainRequestStatus.IN_REVIEW, user=_creator) domain_request.approve() @@ -3133,13 +3121,12 @@ class TestDomainInformationAdmin(TestCase): # == Check for the creator == # - # Check for the right title, email, and phone number in the response. + # Check for the right title and phone number in the response. # We only need to check for the end tag # (Otherwise this test will fail if we change classes, etc) expected_creator_fields = [ # Field, expected value ("title", "Treat inspector"), - ("email", "meoward.jones@igorville.gov"), ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) @@ -4067,8 +4054,8 @@ class TestContactAdmin(TestCase): self.assertEqual(readonly_fields, expected_fields) def test_change_view_for_joined_contact_five_or_less(self): - """Create a contact, join it to 4 domain requests. The 5th join will be a user. - Assert that the warning on the contact form lists 5 joins.""" + """Create a contact, join it to 4 domain requests. 5th join is user. + Assert that the warning on the contact form lists 4 joins.""" with less_console_noise(): self.client.force_login(self.superuser) @@ -4099,17 +4086,17 @@ class TestContactAdmin(TestCase): "
  • Joined to DomainRequest: city4.gov
    • " "
  • Joined to User: staff@example.com
    • " + f"user/{self.staffuser.pk}/change/'>first last staff@example.com" "", ) def test_change_view_for_joined_contact_five_or_more(self): - """Create a contact, join it to 5 domain requests. The 6th join will be a user. + """Create a contact, join it to 5 domain requests. 6th join is user. Assert that the warning on the contact form lists 5 joins and a '1 more' ellispsis.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model - # join it to 5 domain requests. The 6th join will be a user. + # join it to 6 domain requests. contact, _ = Contact.objects.get_or_create(user=self.staffuser) domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 37fdc5354..d7a8dc6f7 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -1209,24 +1209,14 @@ class TestUser(TestCase): # test with a user with contact info defined self.assertTrue(self.user.has_contact_info()) # test with a user without contact info defined - self.user.contact.title = None - self.user.contact.email = None - self.user.contact.phone = None + self.user.title = None + self.user.email = None + self.user.phone = None self.assertFalse(self.user.has_contact_info()) class TestContact(TestCase): def setUp(self): - self.email_for_invalid = "intern@igorville.gov" - self.invalid_user, _ = User.objects.get_or_create( - username=self.email_for_invalid, - email=self.email_for_invalid, - first_name="", - last_name="", - phone="", - ) - self.invalid_contact, _ = Contact.objects.get_or_create(user=self.invalid_user) - self.email = "mayor@igorville.gov" self.user, _ = User.objects.get_or_create( email=self.email, first_name="Jeff", last_name="Lebowski", phone="123456789" @@ -1242,87 +1232,6 @@ class TestContact(TestCase): Contact.objects.all().delete() User.objects.all().delete() - def test_saving_contact_updates_user_first_last_names_and_phone(self): - """When a contact is updated, we propagate the changes to the linked user if it exists.""" - - # User and Contact are created and linked as expected. - # An empty User object should create an empty contact. - self.assertEqual(self.invalid_contact.first_name, "") - self.assertEqual(self.invalid_contact.last_name, "") - self.assertEqual(self.invalid_contact.phone, "") - self.assertEqual(self.invalid_user.first_name, "") - self.assertEqual(self.invalid_user.last_name, "") - self.assertEqual(self.invalid_user.phone, "") - - # Manually update the contact - mimicking production (pre-existing data) - self.invalid_contact.first_name = "Joey" - self.invalid_contact.last_name = "Baloney" - self.invalid_contact.phone = "123456789" - self.invalid_contact.save() - - # Refresh the user object to reflect the changes made in the database - self.invalid_user.refresh_from_db() - - # Updating the contact's first and last names propagate to the user - self.assertEqual(self.invalid_contact.first_name, "Joey") - self.assertEqual(self.invalid_contact.last_name, "Baloney") - self.assertEqual(self.invalid_contact.phone, "123456789") - self.assertEqual(self.invalid_user.first_name, "Joey") - self.assertEqual(self.invalid_user.last_name, "Baloney") - self.assertEqual(self.invalid_user.phone, "123456789") - - def test_saving_contact_does_not_update_user_first_last_names_and_phone(self): - """When a contact is updated, we avoid propagating the changes to the linked user if it already has a value""" - - # User and Contact are created and linked as expected - self.assertEqual(self.contact.first_name, "Jeff") - self.assertEqual(self.contact.last_name, "Lebowski") - self.assertEqual(self.contact.phone, "123456789") - self.assertEqual(self.user.first_name, "Jeff") - self.assertEqual(self.user.last_name, "Lebowski") - self.assertEqual(self.user.phone, "123456789") - - self.contact.first_name = "Joey" - self.contact.last_name = "Baloney" - self.contact.phone = "987654321" - self.contact.save() - - # Refresh the user object to reflect the changes made in the database - self.user.refresh_from_db() - - # Updating the contact's first and last names propagate to the user - self.assertEqual(self.contact.first_name, "Joey") - self.assertEqual(self.contact.last_name, "Baloney") - self.assertEqual(self.contact.phone, "987654321") - self.assertEqual(self.user.first_name, "Jeff") - self.assertEqual(self.user.last_name, "Lebowski") - self.assertEqual(self.user.phone, "123456789") - - def test_saving_contact_does_not_update_user_email(self): - """When a contact's email is updated, the change is not propagated to the user.""" - self.contact.email = "joey.baloney@diaperville.com" - self.contact.save() - - # Refresh the user object to reflect the changes made in the database - self.user.refresh_from_db() - - # Updating the contact's email does not propagate - self.assertEqual(self.contact.email, "joey.baloney@diaperville.com") - self.assertEqual(self.user.email, "mayor@igorville.gov") - - def test_saving_contact_does_not_update_user_email_when_none(self): - """When a contact's email is updated, and the first/last name is none, - the change is not propagated to the user.""" - self.invalid_contact.email = "joey.baloney@diaperville.com" - self.invalid_contact.save() - - # Refresh the user object to reflect the changes made in the database - self.invalid_user.refresh_from_db() - - # Updating the contact's email does not propagate - self.assertEqual(self.invalid_contact.email, "joey.baloney@diaperville.com") - self.assertEqual(self.invalid_user.email, "intern@igorville.gov") - def test_has_more_than_one_join(self): """Test the Contact model method, has_more_than_one_join""" # test for a contact which has one user defined @@ -1334,6 +1243,7 @@ class TestContact(TestCase): def test_has_contact_info(self): """Test that has_contact_info properly returns""" + self.contact.title = "Title" # test with a contact with contact info defined self.assertTrue(self.contact.has_contact_info()) # test with a contact without contact info defined diff --git a/src/registrar/tests/test_signals.py b/src/registrar/tests/test_signals.py deleted file mode 100644 index e796bd12a..000000000 --- a/src/registrar/tests/test_signals.py +++ /dev/null @@ -1,100 +0,0 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model -from registrar.models import Contact - - -class TestUserPostSave(TestCase): - def setUp(self): - self.username = "test_user" - self.first_name = "First" - self.last_name = "Last" - self.email = "info@example.com" - self.phone = "202-555-0133" - - self.preferred_first_name = "One" - self.preferred_last_name = "Two" - self.preferred_email = "front_desk@example.com" - self.preferred_phone = "202-555-0134" - - def test_user_created_without_matching_contact(self): - """Expect 1 Contact containing data copied from User.""" - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - phone=self.phone, - ) - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) - - def test_user_created_with_matching_contact(self): - """Expect 1 Contact associated, but with no data copied from User.""" - self.assertEqual(len(Contact.objects.all()), 0) - Contact.objects.create( - first_name=self.preferred_first_name, - last_name=self.preferred_last_name, - email=self.email, # must be the same, to find the match! - phone=self.preferred_phone, - ) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - ) - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.preferred_first_name) - self.assertEqual(actual.last_name, self.preferred_last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.preferred_phone) - - def test_user_updated_without_matching_contact(self): - """Expect 1 Contact containing data copied from User.""" - # create the user - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create(username=self.username, first_name="", last_name="", email="", phone="") - # delete the contact - Contact.objects.all().delete() - self.assertEqual(len(Contact.objects.all()), 0) - # modify the user - user.username = self.username - user.first_name = self.first_name - user.last_name = self.last_name - user.email = self.email - user.phone = self.phone - user.save() - # test - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) - - def test_user_updated_with_matching_contact(self): - """Expect 1 Contact associated, but with no data copied from User.""" - # create the user - self.assertEqual(len(Contact.objects.all()), 0) - user = get_user_model().objects.create( - username=self.username, - first_name=self.first_name, - last_name=self.last_name, - email=self.email, - phone=self.phone, - ) - # modify the user - user.first_name = self.preferred_first_name - user.last_name = self.preferred_last_name - user.email = self.preferred_email - user.phone = self.preferred_phone - user.save() - # test - actual = Contact.objects.get(user=user) - self.assertEqual(actual.first_name, self.first_name) - self.assertEqual(actual.last_name, self.last_name) - self.assertEqual(actual.email, self.email) - self.assertEqual(actual.phone, self.phone) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 2ac0caf2c..54581ad65 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -57,12 +57,10 @@ class TestWithUser(MockEppLib): last_name = "Last" email = "info@example.com" phone = "8003111234" - self.user = get_user_model().objects.create( - username=username, first_name=first_name, last_name=last_name, email=email, phone=phone - ) title = "test title" - self.user.contact.title = title - self.user.contact.save() + self.user = get_user_model().objects.create( + username=username, first_name=first_name, last_name=last_name, title=title, email=email, phone=phone + ) username_regular_incomplete = "test_regular_user_incomplete" username_other_incomplete = "test_other_user_incomplete" @@ -560,7 +558,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): self.assertContains(finish_setup_page, "Enter your phone number.") # Check for the name of the save button - self.assertContains(finish_setup_page, "contact_setup_save_button") + self.assertContains(finish_setup_page, "user_setup_save_button") # Add a phone number finish_setup_form = finish_setup_page.form @@ -598,7 +596,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): self.assertContains(finish_setup_page, "Enter your phone number.") # Check for the name of the save button - self.assertContains(finish_setup_page, "contact_setup_save_button") + self.assertContains(finish_setup_page, "user_setup_save_button") # Add a phone number finish_setup_form = finish_setup_page.form @@ -613,7 +611,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): # Submit the form using the specific submit button to execute the redirect completed_setup_page = self._submit_form_webtest( - finish_setup_form, follow=True, name="contact_setup_submit_button" + finish_setup_form, follow=True, name="user_setup_submit_button" ) self.assertEqual(completed_setup_page.status_code, 200) diff --git a/src/registrar/tests/test_views_domain.py b/src/registrar/tests/test_views_domain.py index c3bcb02db..f62605d69 100644 --- a/src/registrar/tests/test_views_domain.py +++ b/src/registrar/tests/test_views_domain.py @@ -1478,8 +1478,8 @@ class TestDomainContactInformation(TestDomainOverview): def test_domain_your_contact_information_content(self): """Logged-in user's contact information appears on the page.""" - self.user.contact.first_name = "Testy" - self.user.contact.save() + self.user.first_name = "Testy" + self.user.save() page = self.app.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain.id})) self.assertContains(page, "Testy") diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index b7ce06cce..d6436b058 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -40,7 +40,7 @@ from registrar.models.utility.contact_error import ContactError from registrar.views.utility.permission_views import UserDomainRolePermissionDeleteView from ..forms import ( - ContactForm, + UserForm, AuthorizingOfficialContactForm, DomainOrgNameAddressForm, DomainAddUserForm, @@ -573,7 +573,7 @@ class DomainYourContactInformationView(DomainFormBaseView): """Domain your contact information editing view.""" template_name = "domain_your_contact_information.html" - form_class = ContactForm + form_class = UserForm @waffle_flag("!profile_feature") # type: ignore def dispatch(self, request, *args, **kwargs): # type: ignore @@ -582,7 +582,7 @@ class DomainYourContactInformationView(DomainFormBaseView): def get_form_kwargs(self, *args, **kwargs): """Add domain_info.submitter instance to make a bound form.""" form_kwargs = super().get_form_kwargs(*args, **kwargs) - form_kwargs["instance"] = self.request.user.contact + form_kwargs["instance"] = self.request.user return form_kwargs def get_success_url(self): diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index 3f9aeb79f..e378b9180 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -9,9 +9,6 @@ from django.http import QueryDict from django.views.generic.edit import FormMixin from registrar.forms.user_profile import UserProfileForm, FinishSetupProfileForm from django.urls import NoReverseMatch, reverse -from registrar.models import ( - Contact, -) from registrar.models.user import User from registrar.models.utility.generic_helper import replace_url_queryparams from registrar.views.utility.permission_views import UserProfilePermissionView @@ -25,7 +22,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): Base View for the User Profile. Handles getting and setting the User Profile """ - model = Contact + model = User template_name = "profile.html" form_class = UserProfileForm base_view_name = "user-profile" @@ -57,6 +54,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_context_data(self, **kwargs): """Extend get_context_data to include has_profile_feature_flag""" context = super().get_context_data(**kwargs) + logger.info("UserProfileView::get_context_data") # This is a django waffle flag which toggles features based off of the "flag" table context["has_profile_feature_flag"] = flag_is_active(self.request, "profile_feature") @@ -123,9 +121,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_object(self, queryset=None): """Override get_object to return the logged-in user's contact""" self.user = self.request.user # get the logged in user - if hasattr(self.user, "contact"): # Check if the user has a contact instance - return self.user.contact - return None + return self.user class FinishProfileSetupView(UserProfileView): @@ -134,7 +130,7 @@ class FinishProfileSetupView(UserProfileView): template_name = "finish_profile_setup.html" form_class = FinishSetupProfileForm - model = Contact + model = User base_view_name = "finish-user-profile-setup" @@ -160,11 +156,11 @@ class FinishProfileSetupView(UserProfileView): # Get the current form and validate it if form.is_valid(): self.redirect_page = False - if "contact_setup_save_button" in request.POST: + if "user_setup_save_button" in request.POST: # Logic for when the 'Save' button is clicked, which indicates # user should stay on this page self.redirect_page = False - elif "contact_setup_submit_button" in request.POST: + elif "user_setup_submit_button" in request.POST: # Logic for when the other button is clicked, which indicates # the user should be taken to the redirect page self.redirect_page = True diff --git a/src/registrar/views/utility/permission_views.py b/src/registrar/views/utility/permission_views.py index d35647af2..db727c26e 100644 --- a/src/registrar/views/utility/permission_views.py +++ b/src/registrar/views/utility/permission_views.py @@ -4,7 +4,7 @@ import abc # abstract base class from django.views.generic import DetailView, DeleteView, TemplateView from registrar.models import Domain, DomainRequest, DomainInvitation -from registrar.models.contact import Contact +from registrar.models.user import User from registrar.models.user_domain_role import UserDomainRole from .mixins import ( @@ -154,9 +154,9 @@ class UserProfilePermissionView(UserProfilePermission, DetailView, abc.ABC): """ # DetailView property for what model this is viewing - model = Contact + model = User # variable name in template context for the model object - context_object_name = "contact" + context_object_name = "user" # Abstract property enforces NotImplementedError on an attribute. @property From 7c1b3ee698dc8fec69d15357759b59113f3a3212 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 15:20:29 -0400 Subject: [PATCH 20/55] fixed test_management_scripts for import_tables script --- src/registrar/tests/test_management_scripts.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/registrar/tests/test_management_scripts.py b/src/registrar/tests/test_management_scripts.py index cebee9994..cfe19b091 100644 --- a/src/registrar/tests/test_management_scripts.py +++ b/src/registrar/tests/test_management_scripts.py @@ -1077,11 +1077,6 @@ class TestImportTables(TestCase): for table_name in table_names: mock_path_exists.assert_any_call(f"{table_name}_1.csv") - # Check that clean_tables is called for Contact - mock_get_model.assert_any_call("registrar", "Contact") - model_mock = mock_get_model.return_value - model_mock.objects.all().delete.assert_called() - # Check that logger.info was called for each successful import for table_name in table_names: mock_logger.info.assert_any_call(f"Successfully imported {table_name}_1.csv into {table_name}") From 80af4a0e6b5a2c15fd7f732e7acbbe775efff3e4 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:41:05 -0600 Subject: [PATCH 21/55] Steps for gpg keys --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 4d231a039..72e951bff 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -70,6 +70,7 @@ when setting up your key in Github. Now test commit signing is working by checking out a branch (`yourname/test-commit-signing`) and making some small change to a file. Commit the change (it should prompt you for your GPG credential) and push it to Github. Look on Github at your branch and ensure the commit is `verified`. +### MacOS **Note:** if you are on a mac and not able to successfully create a signed commit, getting the following error: ```zsh error: gpg failed to sign the data @@ -90,6 +91,11 @@ or source ~/.zshrc ``` +### Windows +If you are using windows, it may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). From there, you should be able to access gpg through the terminal. + +When installing, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. + ## Setting up developer sandbox We have three types of environments: stable, staging, and sandbox. Stable (production)and staging (pre-prod) get deployed via tagged release, and developer sandboxes are given to get.gov developers to mess around in a production-like environment without disrupting stable or staging. Each sandbox is namespaced and will automatically be deployed too when the appropriate branch syntax is used for that space in an open pull request. There are several things you need to setup to make the sandbox work for a developer. From 3580d80b35ad2f30657ae3846ff6e17ff584ee52 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:45:51 -0600 Subject: [PATCH 22/55] Add steps for the cf cli --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 72e951bff..6418713a7 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -16,6 +16,8 @@ assignees: abroddrick There are several tools we use locally that you will need to have. - [ ] [Install the cf CLI v7](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html#pkg-mac) for the ability to deploy + - If you are using Windows, installation information can be found [here](https://github.com/cloudfoundry/cli/wiki/V8-CLI-Installation-Guide#installers-and-compressed-binaries) + - Alternatively, for Windows, [consider using chocolately](https://community.chocolatey.org/packages/cloudfoundry-cli/7.2.0) - [ ] Make sure you have `gpg` >2.1.7. Run `gpg --version` to check. If not, [install gnupg](https://formulae.brew.sh/formula/gnupg) - [ ] Install the [Github CLI](https://cli.github.com/) From 45c7f1aaa614b78838b8b9eb38ff041bd9a94e96 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 16:59:18 -0400 Subject: [PATCH 23/55] removed user from contact model and all associated logic --- src/registrar/admin.py | 26 +- .../copy_names_from_contacts_to_users.py | 242 ------------------ ...registrar_c_user_id_4059c4_idx_and_more.py | 21 ++ src/registrar/models/contact.py | 40 --- src/registrar/tests/test_admin.py | 23 +- .../test_copy_names_from_contacts_to_users.py | 124 --------- src/registrar/tests/test_models.py | 8 +- src/registrar/tests/test_views.py | 31 ++- src/registrar/tests/test_views_request.py | 5 +- src/registrar/views/domain_request.py | 6 +- 10 files changed, 67 insertions(+), 459 deletions(-) delete mode 100644 src/registrar/management/commands/copy_names_from_contacts_to_users.py create mode 100644 src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py delete mode 100644 src/registrar/tests/test_copy_names_from_contacts_to_users.py diff --git a/src/registrar/admin.py b/src/registrar/admin.py index a92e4c695..6f455fc89 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -593,12 +593,6 @@ class ListHeaderAdmin(AuditedAdmin, OrderableFieldsMixin): return filters -class UserContactInline(admin.StackedInline): - """Edit a user's profile on the user page.""" - - model = models.Contact - - class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin): """Custom user admin class to use our inlines.""" @@ -615,8 +609,6 @@ class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin): _meta = Meta() - inlines = [UserContactInline] - list_display = ( "username", "overridden_email_field", @@ -894,30 +886,20 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): list_display = [ "name", "email", - "user_exists", ] # this ordering effects the ordering of results - # in autocomplete_fields for user + # in autocomplete_fields ordering = ["first_name", "last_name", "email"] fieldsets = [ ( None, - {"fields": ["user", "first_name", "middle_name", "last_name", "title", "email", "phone"]}, + {"fields": ["first_name", "middle_name", "last_name", "title", "email", "phone"]}, ) ] - autocomplete_fields = ["user"] - change_form_template = "django/admin/email_clipboard_change_form.html" - def user_exists(self, obj): - """Check if the Contact has a related User""" - return "Yes" if obj.user is not None else "No" - - user_exists.short_description = "Is user" # type: ignore - user_exists.admin_order_field = "user" # type: ignore - # We name the custom prop 'contact' because linter # is not allowing a short_description attr on it # This gets around the linter limitation, for now. @@ -935,9 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields = [ - "user", - ] + analyst_readonly_fields = [] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/management/commands/copy_names_from_contacts_to_users.py b/src/registrar/management/commands/copy_names_from_contacts_to_users.py deleted file mode 100644 index 384029400..000000000 --- a/src/registrar/management/commands/copy_names_from_contacts_to_users.py +++ /dev/null @@ -1,242 +0,0 @@ -import logging -import argparse -import sys - -from django.core.management import BaseCommand - -from registrar.management.commands.utility.terminal_helper import ( - TerminalColors, - TerminalHelper, -) -from registrar.models.contact import Contact -from registrar.models.user import User -from registrar.models.utility.domain_helper import DomainHelper - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = """Copy first and last names from a contact to - a related user if it exists and if its first and last name - properties are null or blank strings.""" - - # ====================================================== - # ===================== ARGUMENTS ===================== - # ====================================================== - def add_arguments(self, parser): - parser.add_argument("--debug", action=argparse.BooleanOptionalAction) - - # ====================================================== - # ===================== PRINTING ====================== - # ====================================================== - def print_debug_mode_statements(self, debug_on: bool): - """Prints additional terminal statements to indicate if --debug - or --limitParse are in use""" - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.OKCYAN} - ----------DEBUG MODE ON---------- - Detailed print statements activated. - {TerminalColors.ENDC} - """, - ) - - def print_summary_of_findings( - self, - skipped_contacts, - eligible_users, - processed_users, - debug_on, - ): - """Prints to terminal a summary of findings from - copying first and last names from contacts to users""" - - total_eligible_users = len(eligible_users) - total_skipped_contacts = len(skipped_contacts) - total_processed_users = len(processed_users) - - logger.info( - f"""{TerminalColors.OKGREEN} - ============= FINISHED =============== - Skipped {total_skipped_contacts} contacts - Found {total_eligible_users} users linked to contacts - Processed {total_processed_users} users - {TerminalColors.ENDC} - """ # noqa - ) - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - ======= DEBUG OUTPUT ======= - Users who have a linked contact: - {eligible_users} - - Processed users (users who have a linked contact and a missing first or last name): - {processed_users} - - ===== SKIPPED CONTACTS ===== - {skipped_contacts} - - {TerminalColors.ENDC} - """, - ) - - # ====================================================== - # =================== USER ===================== - # ====================================================== - def update_user(self, contact: Contact, debug_on: bool): - """Given a contact with a first_name and last_name, find & update an existing - corresponding user if her first_name and last_name are null. - - Returns tuple of eligible (is linked to the contact) and processed - (first and last are blank) users. - """ - - user_exists = User.objects.filter(contact=contact).exists() - if user_exists: - try: - # ----------------------- UPDATE USER ----------------------- - # ---- GET THE USER - eligible_user = User.objects.get(contact=contact) - processed_user = None - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - f"""{TerminalColors.YELLOW} - > Found linked user for contact: - {contact} {contact.email} {contact.first_name} {contact.last_name} - > The linked user is {eligible_user} {eligible_user.username} - {TerminalColors.ENDC}""", # noqa - ) - - # Get the fields that exist on both User and Contact. Excludes id. - common_fields = DomainHelper.get_common_fields(User, Contact) - if "email" in common_fields: - # Don't change the email field. - common_fields.remove("email") - - for field in common_fields: - # Grab the value that contact has stored for this field - new_value = getattr(contact, field) - - # Set it on the user field - setattr(eligible_user, field, new_value) - - eligible_user.save() - processed_user = eligible_user - - return ( - eligible_user, - processed_user, - ) - - except Exception as error: - logger.warning( - f""" - {TerminalColors.FAIL} - !!! ERROR: An exception occured in the - User table for the following user: - {contact.email} {contact.first_name} {contact.last_name} - - Exception is: {error} - ----------TERMINATING----------""" - ) - sys.exit() - else: - return None, None - - # ====================================================== - # ================= PROCESS CONTACTS ================== - # ====================================================== - - def process_contacts( - self, - debug_on, - skipped_contacts=[], - eligible_users=[], - processed_users=[], - ): - for contact in Contact.objects.all(): - TerminalHelper.print_conditional( - debug_on, - f"{TerminalColors.OKCYAN}" - "Processing Contact: " - f"{contact.email}," - f" {contact.first_name}," - f" {contact.last_name}" - f"{TerminalColors.ENDC}", - ) - - # ====================================================== - # ====================== USER ======================= - (eligible_user, processed_user) = self.update_user(contact, debug_on) - - debug_string = "" - if eligible_user: - # ---------------- UPDATED ---------------- - eligible_users.append(contact.email) - debug_string = f"eligible user: {eligible_user}" - if processed_user: - processed_users.append(contact.email) - debug_string = f"processed user: {processed_user}" - else: - skipped_contacts.append(contact.email) - debug_string = f"skipped user: {contact.email}" - - # DEBUG: - TerminalHelper.print_conditional( - debug_on, - (f"{TerminalColors.OKCYAN} {debug_string} {TerminalColors.ENDC}"), - ) - - return ( - skipped_contacts, - eligible_users, - processed_users, - ) - - # ====================================================== - # ===================== HANDLE ======================== - # ====================================================== - def handle( - self, - **options, - ): - """Parse entries in Contact table - and update valid corresponding entries in the - User table.""" - - # grab command line arguments and store locally... - debug_on = options.get("debug") - - self.print_debug_mode_statements(debug_on) - - logger.info( - f"""{TerminalColors.OKCYAN} - ========================== - Beginning Data Transfer - ========================== - {TerminalColors.ENDC}""" - ) - - logger.info( - f"""{TerminalColors.OKCYAN} - ========= Adding Domains and Domain Invitations ========= - {TerminalColors.ENDC}""" - ) - ( - skipped_contacts, - eligible_users, - processed_users, - ) = self.process_contacts( - debug_on, - ) - - self.print_summary_of_findings( - skipped_contacts, - eligible_users, - processed_users, - debug_on, - ) diff --git a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py b/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py new file mode 100644 index 000000000..858210be7 --- /dev/null +++ b/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.10 on 2024-07-02 19:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("registrar", "0109_domaininformation_sub_organization_and_more"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="contact", + name="registrar_c_user_id_4059c4_idx", + ), + migrations.RemoveField( + model_name="contact", + name="user", + ), + ] diff --git a/src/registrar/models/contact.py b/src/registrar/models/contact.py index f7bae3491..903633749 100644 --- a/src/registrar/models/contact.py +++ b/src/registrar/models/contact.py @@ -14,17 +14,9 @@ class Contact(TimeStampedModel): """Contains meta information about this class""" indexes = [ - models.Index(fields=["user"]), models.Index(fields=["email"]), ] - user = models.OneToOneField( - "registrar.User", - null=True, - blank=True, - on_delete=models.SET_NULL, - ) - first_name = models.CharField( null=True, blank=True, @@ -103,38 +95,6 @@ class Contact(TimeStampedModel): def has_contact_info(self): return bool(self.title or self.email or self.phone) - def save(self, *args, **kwargs): - # Call the parent class's save method to perform the actual save - super().save(*args, **kwargs) - - if self.user: - updated = False - - # Update first name and last name if necessary - if not self.user.first_name or not self.user.last_name: - self.user.first_name = self.first_name - self.user.last_name = self.last_name - updated = True - - # Update middle_name if necessary - if not self.user.middle_name: - self.user.middle_name = self.middle_name - updated = True - - # Update phone if necessary - if not self.user.phone: - self.user.phone = self.phone - updated = True - - # Update title if necessary - if not self.user.title: - self.user.title = self.title - updated = True - - # Save user if any updates were made - if updated: - self.user.save() - def __str__(self): if self.first_name or self.last_name: return self.get_formatted_name() diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 65427bb65..a1532cd39 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -4036,9 +4036,7 @@ class TestContactAdmin(TestCase): readonly_fields = self.admin.get_readonly_fields(request) - expected_fields = [ - "user", - ] + expected_fields = [] self.assertEqual(readonly_fields, expected_fields) @@ -4054,15 +4052,18 @@ class TestContactAdmin(TestCase): self.assertEqual(readonly_fields, expected_fields) def test_change_view_for_joined_contact_five_or_less(self): - """Create a contact, join it to 4 domain requests. 5th join is user. + """Create a contact, join it to 4 domain requests. Assert that the warning on the contact form lists 4 joins.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson", + ) - # join it to 4 domain requests. The 5th join will be a user. + # join it to 4 domain requests. domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") @@ -4085,24 +4086,26 @@ class TestContactAdmin(TestCase): f"domainrequest/{domain_request3.pk}/change/'>city3.gov" "
  • Joined to DomainRequest: city4.gov
    • " - "
  • Joined to User: first last staff@example.com
    • " "", ) def test_change_view_for_joined_contact_five_or_more(self): - """Create a contact, join it to 5 domain requests. 6th join is user. + """Create a contact, join it to 6 domain requests. Assert that the warning on the contact form lists 5 joins and a '1 more' ellispsis.""" with less_console_noise(): self.client.force_login(self.superuser) # Create an instance of the model # join it to 6 domain requests. - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson", + ) domain_request1 = completed_domain_request(submitter=contact, name="city1.gov") domain_request2 = completed_domain_request(submitter=contact, name="city2.gov") domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") domain_request4 = completed_domain_request(submitter=contact, name="city4.gov") domain_request5 = completed_domain_request(submitter=contact, name="city5.gov") + domain_request6 = completed_domain_request(submitter=contact, name="city6.gov") with patch("django.contrib.messages.warning") as mock_warning: # Use the test client to simulate the request response = self.client.get(reverse("admin:registrar_contact_change", args=[contact.pk])) diff --git a/src/registrar/tests/test_copy_names_from_contacts_to_users.py b/src/registrar/tests/test_copy_names_from_contacts_to_users.py deleted file mode 100644 index 7fcbede1e..000000000 --- a/src/registrar/tests/test_copy_names_from_contacts_to_users.py +++ /dev/null @@ -1,124 +0,0 @@ -from django.test import TestCase - -from registrar.models import ( - User, - Contact, -) - -from registrar.management.commands.copy_names_from_contacts_to_users import Command - - -class TestDataUpdates(TestCase): - def setUp(self): - """We cannot setup the user details because contacts will override the first and last names in its save method - so we will initiate the users, setup the contacts and link them, and leave the rest of the setup to the test(s). - """ - - self.user1 = User.objects.create(username="user1") - self.user2 = User.objects.create(username="user2") - self.user3 = User.objects.create(username="user3") - self.user4 = User.objects.create(username="user4") - # The last user created triggers the creation of a contact and attaches itself to it. See signals. - # This bs_user defuses that situation. - self.bs_user = User.objects.create() - - self.contact1 = Contact.objects.create( - user=self.user1, - email="email1@igorville.gov", - first_name="first1", - last_name="last1", - middle_name="middle1", - title="title1", - ) - self.contact2 = Contact.objects.create( - user=self.user2, - email="email2@igorville.gov", - first_name="first2", - last_name="last2", - middle_name="middle2", - title="title2", - ) - self.contact3 = Contact.objects.create( - user=self.user3, - email="email3@igorville.gov", - first_name="first3", - last_name="last3", - middle_name="middle3", - title="title3", - ) - self.contact4 = Contact.objects.create( - email="email4@igorville.gov", first_name="first4", last_name="last4", middle_name="middle4", title="title4" - ) - - self.command = Command() - - def tearDown(self): - """Clean up""" - # Delete users and contacts - User.objects.all().delete() - Contact.objects.all().delete() - - def test_script_updates_linked_users(self): - """Test the script that copies contact information to the user object""" - - # Set up the users' first and last names here so - # they that they don't get overwritten by Contact's save() - # User with no first or last names - self.user1.first_name = "" - self.user1.last_name = "" - self.user1.title = "dummytitle" - self.user1.middle_name = "dummymiddle" - self.user1.save() - - # User with a first name but no last name - self.user2.first_name = "First name but no last name" - self.user2.last_name = "" - self.user2.save() - - # User with a first and last name - self.user3.first_name = "An existing first name" - self.user3.last_name = "An existing last name" - self.user3.save() - - # Call the parent method the same way we do it in the script - skipped_contacts = [] - eligible_users = [] - processed_users = [] - ( - skipped_contacts, - eligible_users, - processed_users, - ) = self.command.process_contacts( - # Set debugging to False - False, - skipped_contacts, - eligible_users, - processed_users, - ) - - # Trigger DB refresh - self.user1.refresh_from_db() - self.user2.refresh_from_db() - self.user3.refresh_from_db() - - # Asserts - # The user that has no first and last names will get them from the contact - self.assertEqual(self.user1.first_name, "first1") - self.assertEqual(self.user1.last_name, "last1") - self.assertEqual(self.user1.middle_name, "middle1") - self.assertEqual(self.user1.title, "title1") - # The user that has a first but no last will be updated - self.assertEqual(self.user2.first_name, "first2") - self.assertEqual(self.user2.last_name, "last2") - self.assertEqual(self.user2.middle_name, "middle2") - self.assertEqual(self.user2.title, "title2") - # The user that has a first and a last will be updated - self.assertEqual(self.user3.first_name, "first3") - self.assertEqual(self.user3.last_name, "last3") - self.assertEqual(self.user3.middle_name, "middle3") - self.assertEqual(self.user3.title, "title3") - # The unlinked user will be left alone - self.assertEqual(self.user4.first_name, "") - self.assertEqual(self.user4.last_name, "") - self.assertEqual(self.user4.middle_name, None) - self.assertEqual(self.user4.title, None) diff --git a/src/registrar/tests/test_models.py b/src/registrar/tests/test_models.py index 9a29f5428..0c5ca9193 100644 --- a/src/registrar/tests/test_models.py +++ b/src/registrar/tests/test_models.py @@ -1221,7 +1221,10 @@ class TestContact(TestCase): self.user, _ = User.objects.get_or_create( email=self.email, first_name="Jeff", last_name="Lebowski", phone="123456789" ) - self.contact, _ = Contact.objects.get_or_create(user=self.user) + self.contact, _ = Contact.objects.get_or_create( + first_name="Jeff", + last_name="Lebowski", + ) self.contact_as_so, _ = Contact.objects.get_or_create(email="newguy@igorville.gov") self.domain_request = DomainRequest.objects.create(creator=self.user, senior_official=self.contact_as_so) @@ -1234,9 +1237,6 @@ class TestContact(TestCase): def test_has_more_than_one_join(self): """Test the Contact model method, has_more_than_one_join""" - # test for a contact which has one user defined - self.assertFalse(self.contact.has_more_than_one_join("user")) - self.assertTrue(self.contact.has_more_than_one_join("senior_official")) # test for a contact which is assigned as a senior official on a domain request self.assertFalse(self.contact_as_so.has_more_than_one_join("senior_official")) self.assertTrue(self.contact_as_so.has_more_than_one_join("submitted_domain_requests")) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 42c7b2d38..b98a30e79 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -372,7 +372,10 @@ class HomeTests(TestWithUser): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( @@ -405,17 +408,12 @@ class HomeTests(TestWithUser): igorville = DomainRequest.objects.filter(requested_domain__name="igorville.gov") self.assertFalse(igorville.exists()) - # Check if the orphaned contact was deleted + # Check if the orphaned contacts were deleted orphan = Contact.objects.filter(id=contact.id) self.assertFalse(orphan.exists()) + orphan = Contact.objects.filter(id=contact_user.id) + self.assertFalse(orphan.exists()) - # All non-orphan contacts should still exist and are unaltered - try: - current_user = Contact.objects.filter(id=contact_user.id).get() - except Contact.DoesNotExist: - self.fail("contact_user (a non-orphaned contact) was deleted") - - self.assertEqual(current_user, contact_user) try: edge_case = Contact.objects.filter(id=contact_2.id).get() except Contact.DoesNotExist: @@ -444,7 +442,10 @@ class HomeTests(TestWithUser): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( @@ -863,7 +864,10 @@ class UserProfileTests(TestWithUser, WebTest): def test_request_when_profile_feature_on(self): """test that Your profile is in request page when profile feature is on""" - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakerson", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( creator=self.user, @@ -882,7 +886,10 @@ class UserProfileTests(TestWithUser, WebTest): def test_request_when_profile_feature_off(self): """test that Your profile is not in request page when profile feature is off""" - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakerson", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( creator=self.user, diff --git a/src/registrar/tests/test_views_request.py b/src/registrar/tests/test_views_request.py index e78dfe860..de924576b 100644 --- a/src/registrar/tests/test_views_request.py +++ b/src/registrar/tests/test_views_request.py @@ -2974,7 +2974,10 @@ class TestWizardUnlockingSteps(TestWithUser, WebTest): ) # Attach a user object to a contact (should not be deleted) - contact_user, _ = Contact.objects.get_or_create(user=self.user) + contact_user, _ = Contact.objects.get_or_create( + first_name="Hank", + last_name="McFakey", + ) site = DraftDomain.objects.create(name="igorville.gov") domain_request = DomainRequest.objects.create( diff --git a/src/registrar/views/domain_request.py b/src/registrar/views/domain_request.py index e8e82500e..a7d6aa6ae 100644 --- a/src/registrar/views/domain_request.py +++ b/src/registrar/views/domain_request.py @@ -812,15 +812,15 @@ class DomainRequestDeleteView(DomainRequestPermissionDeleteView): self.object = self.get_object() self.object.delete() - # Delete orphaned contacts - but only for if they are not associated with a user - Contact.objects.filter(id__in=contacts_to_delete, user=None).delete() + # Delete orphaned contacts + Contact.objects.filter(id__in=contacts_to_delete).delete() # After a delete occurs, do a second sweep on any returned duplicates. # This determines if any of these three fields share a contact, which is used for # the edge case where the same user may be an SO, and a submitter, for example. if len(duplicates) > 0: duplicates_to_delete, _ = self._get_orphaned_contacts(domain_request, check_db=True) - Contact.objects.filter(id__in=duplicates_to_delete, user=None).delete() + Contact.objects.filter(id__in=duplicates_to_delete).delete() # Return a 200 response with an empty body return HttpResponse(status=200) From c964280195a2ff135578b3a7fc19b924db81a79c Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 2 Jul 2024 15:04:34 -0600 Subject: [PATCH 24/55] Updated migrations --- ...al.py => 0110_seniorofficial_portfolio_senior_official.py} | 4 ++-- .../{0110_create_groups_v15.py => 0111_create_groups_v15.py} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/registrar/migrations/{0109_seniorofficial_portfolio_senior_official.py => 0110_seniorofficial_portfolio_senior_official.py} (92%) rename src/registrar/migrations/{0110_create_groups_v15.py => 0111_create_groups_v15.py} (95%) diff --git a/src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py b/src/registrar/migrations/0110_seniorofficial_portfolio_senior_official.py similarity index 92% rename from src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py rename to src/registrar/migrations/0110_seniorofficial_portfolio_senior_official.py index 9e4d0b7b1..c344898c3 100644 --- a/src/registrar/migrations/0109_seniorofficial_portfolio_senior_official.py +++ b/src/registrar/migrations/0110_seniorofficial_portfolio_senior_official.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-07-02 14:59 +# Generated by Django 4.2.10 on 2024-07-02 21:03 from django.db import migrations, models import django.db.models.deletion @@ -8,7 +8,7 @@ import phonenumber_field.modelfields class Migration(migrations.Migration): dependencies = [ - ("registrar", "0108_domaininformation_authorizing_official_and_more"), + ("registrar", "0109_domaininformation_sub_organization_and_more"), ] operations = [ diff --git a/src/registrar/migrations/0110_create_groups_v15.py b/src/registrar/migrations/0111_create_groups_v15.py similarity index 95% rename from src/registrar/migrations/0110_create_groups_v15.py rename to src/registrar/migrations/0111_create_groups_v15.py index bf7c7f325..6b21f4b0d 100644 --- a/src/registrar/migrations/0110_create_groups_v15.py +++ b/src/registrar/migrations/0111_create_groups_v15.py @@ -25,7 +25,7 @@ def create_groups(apps, schema_editor) -> Any: class Migration(migrations.Migration): dependencies = [ - ("registrar", "0109_seniorofficial_portfolio_senior_official"), + ("registrar", "0110_seniorofficial_portfolio_senior_official"), ] operations = [ From 1dc3bf883e299cd89df521b8ccad2163bcfc6976 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 17:10:52 -0400 Subject: [PATCH 25/55] updated for linter --- src/registrar/admin.py | 2 +- src/registrar/tests/test_admin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 6f455fc89..967c36494 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -917,7 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields = [] + analyst_readonly_fields: list[str] = [] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index a1532cd39..889a2c0e0 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -4105,7 +4105,7 @@ class TestContactAdmin(TestCase): domain_request3 = completed_domain_request(submitter=contact, name="city3.gov") domain_request4 = completed_domain_request(submitter=contact, name="city4.gov") domain_request5 = completed_domain_request(submitter=contact, name="city5.gov") - domain_request6 = completed_domain_request(submitter=contact, name="city6.gov") + completed_domain_request(submitter=contact, name="city6.gov") with patch("django.contrib.messages.warning") as mock_warning: # Use the test client to simulate the request response = self.client.get(reverse("admin:registrar_contact_change", args=[contact.pk])) From 6076400e7cf719ac606a4ae69ae489cdcdfb469f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 18:43:46 -0400 Subject: [PATCH 26/55] a number of small refactors and cleanup of comments --- src/registrar/utility/csv_export.py | 36 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index f2786c2cd..cfde68a32 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -11,9 +11,7 @@ from registrar.models import ( PublicContact, UserDomainRole, ) -from django.db.models import QuerySet, Value, CharField, Count, Q, F -from django.db.models import Case, When, DateField -from django.db.models import ManyToManyField +from django.db.models import Case, CharField, Count, DateField, F, ManyToManyField, Q, QuerySet, Value, When from django.utils import timezone from django.db.models.functions import Concat, Coalesce from django.contrib.postgres.aggregates import StringAgg @@ -105,7 +103,7 @@ class BaseExport(ABC): @classmethod def get_exclusions(cls): """ - Get a Q object of exclusion conditions to use when building queryset. + Get a Q object of exclusion conditions to pass to .exclude() when building queryset. """ return Q() @@ -119,7 +117,8 @@ class BaseExport(ABC): @classmethod def get_computed_fields(cls): """ - Get a dict of computed fields. + Get a dict of computed fields. These are fields that do not exist on the model normally + and will be passed to .annotate() when building a queryset. """ return {} @@ -258,6 +257,7 @@ class BaseExport(ABC): writer.writerows(rows) @classmethod + @abstractmethod def parse_row(cls, columns, model): """ Given a set of columns and a model dictionary, generate a new row from cleaned column data. @@ -268,7 +268,8 @@ class BaseExport(ABC): class DomainExport(BaseExport): """ - A collection of functions which return csv files regarding the Domain model. + A collection of functions which return csv files regarding Domains. Although class is + named DomainExport, the base model for the export is DomainInformation. Second class in an inheritance tree of 3. """ @@ -293,21 +294,22 @@ class DomainExport(BaseExport): annotated_domain_infos = [] # Create mapping of domain to a list of invited users and managers - invited_users_dict = defaultdict(list) - for domain, email in domain_invitations: - invited_users_dict[domain].append(email) + # invited_users_dict = defaultdict(list) + # for domain, email in domain_invitations: + # invited_users_dict[domain].append(email) - managers_dict = defaultdict(list) - for domain, email in user_domain_roles: - managers_dict[domain].append(email) + # managers_dict = defaultdict(list) + # for domain, email in user_domain_roles: + # managers_dict[domain].append(email) - # Annotate with security_contact from public_contacts + # Annotate with security_contact from public_contacts, invited users + # from domain_invitations, and managers from user_domain_roles for domain_info in queryset: domain_info["security_contact_email"] = public_contacts.get( domain_info.get("domain__security_contact_registry_id") ) - domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) - domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) + domain_info["invited_users"] = domain_invitations.get(domain_info.get("domain__name")) + domain_info["managers"] = user_domain_roles.get(domain_info.get("domain__name")) annotated_domain_infos.append(domain_info) if annotated_domain_infos: @@ -334,7 +336,7 @@ class DomainExport(BaseExport): Fetch all DomainInvitation entries and return a mapping of domain to email. """ domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") - return list(domain_invitations) + return {domain__name: email for domain__name, email in domain_invitations} @classmethod def get_all_user_domain_roles(cls): @@ -342,7 +344,7 @@ class DomainExport(BaseExport): Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") - return list(user_domain_roles) + return {domain__name: user__email for domain__name, user__email in user_domain_roles} @classmethod def parse_row(cls, columns, model): From e0053fa319c7074ec8dfdda8bcbf1f8443f7671e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 2 Jul 2024 18:52:31 -0400 Subject: [PATCH 27/55] linter --- src/registrar/utility/csv_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index cfde68a32..7f77db5fd 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from collections import defaultdict import csv import logging from datetime import datetime From 5cfdda3e8d847b07281d9f1ca2e33890d180bc8b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:39:23 -0600 Subject: [PATCH 28/55] Add additional instructions --- .github/ISSUE_TEMPLATE/developer-onboarding.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/developer-onboarding.md b/.github/ISSUE_TEMPLATE/developer-onboarding.md index 6418713a7..eb547d5ab 100644 --- a/.github/ISSUE_TEMPLATE/developer-onboarding.md +++ b/.github/ISSUE_TEMPLATE/developer-onboarding.md @@ -94,9 +94,13 @@ source ~/.zshrc ``` ### Windows -If you are using windows, it may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). From there, you should be able to access gpg through the terminal. +If GPG doesn't work out of the box with git for you: +- You can [download the GPG binary directly](https://gnupg.org/download/). +- It may be helpful to use [gpg4win](https://www.gpg4win.org/get-gpg4win.html). -When installing, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. +From there, you should be able to access gpg through the terminal. + +Additionally, consider a gpg key manager like Kleopatra if you run into issues with environment variables or with the gpg service not running on startup. ## Setting up developer sandbox From c7a5b50a45cfd29f8624e96d353179ddd8a71c87 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 3 Jul 2024 15:09:48 -0600 Subject: [PATCH 29/55] fixed 500 error --- src/registrar/views/user_profile.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index 3f9aeb79f..67a0c73ce 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -1,5 +1,4 @@ """Views for a User Profile. - """ import logging @@ -68,7 +67,8 @@ class UserProfileView(UserProfilePermissionView, FormMixin): # Show back button conditional on user having finished setup context["show_back_button"] = False - if hasattr(self.user, "finished_setup") and self.user.finished_setup: + form = self.get_form() + if hasattr(self.user, "finished_setup") and self.user.finished_setup and form.is_valid(): context["user_finished_setup"] = True context["show_back_button"] = True @@ -144,7 +144,8 @@ class FinishProfileSetupView(UserProfileView): # Show back button conditional on user having finished setup context["show_back_button"] = False - if hasattr(self.user, "finished_setup") and self.user.finished_setup: + form = self.get_form() + if hasattr(self.user, "finished_setup") and self.user.finished_setup and form.is_valid(): if kwargs.get("redirect") == "home": context["show_back_button"] = True else: From cdc9f8741d3ff9f9af6d7e6ccaafd196c4d3b8a0 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 3 Jul 2024 15:32:31 -0600 Subject: [PATCH 30/55] fixed typo --- src/registrar/templates/domain_request_intro.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_request_intro.html b/src/registrar/templates/domain_request_intro.html index 370ea2b2b..f4319f53d 100644 --- a/src/registrar/templates/domain_request_intro.html +++ b/src/registrar/templates/domain_request_intro.html @@ -18,7 +18,7 @@ completing your domain request might take around 15 minutes.

    {% if has_profile_feature_flag %}

    How we’ll reach you

    -

    While reviewing your domain request, we may need to reach out with questions. We’ll also email you when we complete our review If the contact information below is not correct, visit your profile to make updates.

    +

    While reviewing your domain request, we may need to reach out with questions. We’ll also email you when we complete our review. If the contact information below is not correct, visit your profile to make updates.

    {% include "includes/profile_information.html" with user=user%} {% endif %} From 9f4ef9f0252b5a7643d9e5901f959a41843796ed Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 3 Jul 2024 16:24:27 -0600 Subject: [PATCH 31/55] fixing tests --- src/registrar/tests/test_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 61bc94a32..b322f20b2 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -602,6 +602,7 @@ class FinishUserProfileTests(TestWithUser, WebTest): # Add a phone number finish_setup_form = finish_setup_page.form + finish_setup_form["first_name"] = "firstname" finish_setup_form["phone"] = "(201) 555-0123" finish_setup_form["title"] = "CEO" finish_setup_form["last_name"] = "example" @@ -730,7 +731,7 @@ class FinishUserProfileForOtherUsersTests(TestWithUser, WebTest): self.assertContains(save_page, "Your profile has been updated.") # We need to assert that logo is not clickable and links to manage your domain are not present - self.assertContains(save_page, "anage your domains", count=2) + self.assertContains(save_page, "manage your domains", count=1) self.assertNotContains( save_page, "Before you can manage your domains, we need you to add contact information" ) From 1146d97835cb7fb49581a16718800b7416ff860a Mon Sep 17 00:00:00 2001 From: Rebecca Hsieh Date: Wed, 3 Jul 2024 16:57:22 -0700 Subject: [PATCH 32/55] Hide the other sections when in rejected state --- src/registrar/templates/domain_request_status.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/registrar/templates/domain_request_status.html b/src/registrar/templates/domain_request_status.html index ad3dc4069..183a8be81 100644 --- a/src/registrar/templates/domain_request_status.html +++ b/src/registrar/templates/domain_request_status.html @@ -42,10 +42,13 @@

    Last updated: {{DomainRequest.updated_at|date:"F j, Y"}}

    + + {% if DomainRequest.status != 'rejected' %}

    {% include "includes/domain_request.html" %}

    Withdraw request

    + {% endif %}
    From 195d3287f3f105e51bb41ea3607dc6036f4c8ec6 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 3 Jul 2024 18:11:26 -0600 Subject: [PATCH 33/55] Fix attempt #2 --- src/registrar/tests/test_views.py | 4 +++- src/registrar/views/user_profile.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index b322f20b2..7a375cd11 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -731,7 +731,9 @@ class FinishUserProfileForOtherUsersTests(TestWithUser, WebTest): self.assertContains(save_page, "Your profile has been updated.") # We need to assert that logo is not clickable and links to manage your domain are not present - self.assertContains(save_page, "manage your domains", count=1) + # NOTE: "anage" is not a typo. It is to accomodate the fact that the "m" is uppercase in one + # instance and lowercase in the other. + self.assertContains(save_page, "anage your domains", count=2) self.assertNotContains( save_page, "Before you can manage your domains, we need you to add contact information" ) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index 67a0c73ce..b3718bd20 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -67,8 +67,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): # Show back button conditional on user having finished setup context["show_back_button"] = False - form = self.get_form() - if hasattr(self.user, "finished_setup") and self.user.finished_setup and form.is_valid(): + if hasattr(self.user, "finished_setup") and self.user.finished_setup: context["user_finished_setup"] = True context["show_back_button"] = True @@ -120,7 +119,17 @@ class UserProfileView(UserProfilePermissionView, FormMixin): # superclass has the redirect return super().form_valid(form) + def form_invalid(self, form): + """If the form is invalid, conditionally display an additional error.""" + if hasattr(self.user, "finished_setup") and not self.user.finished_setup: + messages.error(self.request, "Before you can manage your domain, we need you to add contact information.") + logger.info("got here, now rendering response") + form.initial['redirect'] = form.data.get('redirect') + logger.info(form.initial) + return super().form_invalid(form) + def get_object(self, queryset=None): + """Override get_object to return the logged-in user's contact""" self.user = self.request.user # get the logged in user if hasattr(self.user, "contact"): # Check if the user has a contact instance @@ -144,8 +153,7 @@ class FinishProfileSetupView(UserProfileView): # Show back button conditional on user having finished setup context["show_back_button"] = False - form = self.get_form() - if hasattr(self.user, "finished_setup") and self.user.finished_setup and form.is_valid(): + if hasattr(self.user, "finished_setup") and self.user.finished_setup: if kwargs.get("redirect") == "home": context["show_back_button"] = True else: @@ -182,4 +190,4 @@ class FinishProfileSetupView(UserProfileView): return reverse(redirect_param) except NoReverseMatch as err: logger.error(f"get_redirect_url -> Could not find the specified page. Err: {err}") - return super().get_success_url() + return super().get_success_url() \ No newline at end of file From f0af94978304b2a9a1bff99af8c182b6b8852ce5 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Wed, 3 Jul 2024 18:42:49 -0600 Subject: [PATCH 34/55] linted --- src/registrar/views/user_profile.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index b3718bd20..feb545e12 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -108,6 +108,8 @@ class UserProfileView(UserProfilePermissionView, FormMixin): """If the form is invalid, conditionally display an additional error.""" if hasattr(self.user, "finished_setup") and not self.user.finished_setup: messages.error(self.request, "Before you can manage your domain, we need you to add contact information.") + form.initial["redirect"] = form.data.get("redirect") + logger.info(form.initial) return super().form_invalid(form) def form_valid(self, form): @@ -119,17 +121,7 @@ class UserProfileView(UserProfilePermissionView, FormMixin): # superclass has the redirect return super().form_valid(form) - def form_invalid(self, form): - """If the form is invalid, conditionally display an additional error.""" - if hasattr(self.user, "finished_setup") and not self.user.finished_setup: - messages.error(self.request, "Before you can manage your domain, we need you to add contact information.") - logger.info("got here, now rendering response") - form.initial['redirect'] = form.data.get('redirect') - logger.info(form.initial) - return super().form_invalid(form) - def get_object(self, queryset=None): - """Override get_object to return the logged-in user's contact""" self.user = self.request.user # get the logged in user if hasattr(self.user, "contact"): # Check if the user has a contact instance @@ -190,4 +182,4 @@ class FinishProfileSetupView(UserProfileView): return reverse(redirect_param) except NoReverseMatch as err: logger.error(f"get_redirect_url -> Could not find the specified page. Err: {err}") - return super().get_success_url() \ No newline at end of file + return super().get_success_url() From d6d178e4e156f3cd7ad30c90eba074aca4016095 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 08:52:15 -0400 Subject: [PATCH 35/55] updated authorizing official to senior official --- src/registrar/tests/test_reports.py | 20 +++++----- src/registrar/utility/csv_export.py | 58 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 0015ae84f..265876987 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -207,7 +207,7 @@ class ExportDataTest(MockDb, MockEppLib): @less_console_noise_decorator def test_domain_data_type(self): - """Shows security contacts, domain managers, ao""" + """Shows security contacts, domain managers, so""" # Add security email information self.domain_1.name = "defaultsecurity.gov" self.domain_1.save() @@ -231,8 +231,8 @@ class ExportDataTest(MockDb, MockEppLib): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,AO," - "AO email,Security contact email,Domain managers,Invited domain managers\n" + "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," + "SO email,Security contact email,Domain managers,Invited domain managers\n" "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,, ,,," "meoward@rocks.com,\n" "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," @@ -534,10 +534,10 @@ class ExportDataTest(MockDb, MockEppLib): "Creator approved domains count", "Creator active requests count", "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", + "SO first name", + "SO last name", + "SO email", + "SO title/role", "Request purpose", "Request additional details", "Other contacts", @@ -560,8 +560,8 @@ class ExportDataTest(MockDb, MockEppLib): "Domain request,Status,Domain type,Federal type," "Federal agency,Organization name,Election office,City,State/territory," "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,AO first name,AO last name,AO email," - "AO title/role,Request purpose,Request additional details,Other contacts," + "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," + "SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," @@ -633,4 +633,4 @@ class HelperFunctions(MockDb): } submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 7f77db5fd..44f8e6662 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -382,8 +382,8 @@ class DomainExport(BaseExport): "Organization name": model.get("organization_name"), "City": model.get("city"), "State": model.get("state_territory"), - "AO": model.get("ao_name"), - "AO email": model.get("authorizing_official__email"), + "SO": model.get("so_name"), + "SO email": model.get("senior_official__email"), "Security contact email": model.get("security_contact_email"), "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -435,7 +435,7 @@ class DomainExport(BaseExport): class DomainDataType(DomainExport): """ - Shows security contacts, domain managers, ao + Shows security contacts, domain managers, so Inherits from BaseExport -> DomainExport """ @@ -454,8 +454,8 @@ class DomainDataType(DomainExport): "Organization name", "City", "State", - "AO", - "AO email", + "SO", + "SO email", "Security contact email", "Domain managers", "Invited domain managers", @@ -502,7 +502,7 @@ class DomainDataType(DomainExport): """ Get a list of tables to pass to select_related when building queryset. """ - return ["domain", "authorizing_official"] + return ["domain", "senior_official"] @classmethod def get_prefetch_related(cls): @@ -517,10 +517,10 @@ class DomainDataType(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -538,7 +538,7 @@ class DomainDataType(DomainExport): "domain__created_at", "domain__deleted", "domain__security_contact_registry_id", - "authorizing_official__email", + "senior_official__email", "federal_agency__agency", ] @@ -618,10 +618,10 @@ class DomainDataFull(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -714,10 +714,10 @@ class DomainDataFederal(DomainExport): Get a dict of computed fields. """ return { - "ao_name": Concat( - Coalesce(F("authorizing_official__first_name"), Value("")), + "so_name": Concat( + Coalesce(F("senior_official__first_name"), Value("")), Value(" "), - Coalesce(F("authorizing_official__last_name"), Value("")), + Coalesce(F("senior_official__last_name"), Value("")), output_field=CharField(), ), } @@ -1181,10 +1181,10 @@ class DomainRequestExport(BaseExport): "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. "Federal agency": model.get("federal_agency__agency"), - "AO first name": model.get("authorizing_official__first_name"), - "AO last name": model.get("authorizing_official__last_name"), - "AO email": model.get("authorizing_official__email"), - "AO title/role": model.get("authorizing_official__title"), + "SO first name": model.get("senior_official__first_name"), + "SO last name": model.get("senior_official__last_name"), + "SO email": model.get("senior_official__email"), + "SO title/role": model.get("senior_official__title"), "Creator first name": model.get("creator__first_name"), "Creator last name": model.get("creator__last_name"), "Creator email": model.get("creator__email"), @@ -1280,10 +1280,10 @@ class DomainRequestDataFull(DomainRequestExport): "Creator approved domains count", "Creator active requests count", "Alternative domains", - "AO first name", - "AO last name", - "AO email", - "AO title/role", + "SO first name", + "SO last name", + "SO email", + "SO title/role", "Request purpose", "Request additional details", "Other contacts", @@ -1297,7 +1297,7 @@ class DomainRequestDataFull(DomainRequestExport): """ Get a list of tables to pass to select_related when building queryset. """ - return ["creator", "authorizing_official", "federal_agency", "investigator", "requested_domain"] + return ["creator", "senior_official", "federal_agency", "investigator", "requested_domain"] @classmethod def get_prefetch_related(cls): @@ -1355,10 +1355,10 @@ class DomainRequestDataFull(DomainRequestExport): return [ "requested_domain__name", "federal_agency__agency", - "authorizing_official__first_name", - "authorizing_official__last_name", - "authorizing_official__email", - "authorizing_official__title", + "senior_official__first_name", + "senior_official__last_name", + "senior_official__email", + "senior_official__title", "creator__first_name", "creator__last_name", "creator__email", From fecaa44cc175dfce9f9eaeab0daf3222368dc9b8 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 09:33:28 -0400 Subject: [PATCH 36/55] fixed domain invitations and domain managers --- src/registrar/utility/csv_export.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 44f8e6662..334742d17 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from collections import defaultdict import csv import logging from datetime import datetime @@ -293,13 +294,13 @@ class DomainExport(BaseExport): annotated_domain_infos = [] # Create mapping of domain to a list of invited users and managers - # invited_users_dict = defaultdict(list) - # for domain, email in domain_invitations: - # invited_users_dict[domain].append(email) + invited_users_dict = defaultdict(list) + for domain, email in domain_invitations: + invited_users_dict[domain].append(email) - # managers_dict = defaultdict(list) - # for domain, email in user_domain_roles: - # managers_dict[domain].append(email) + managers_dict = defaultdict(list) + for domain, email in user_domain_roles: + managers_dict[domain].append(email) # Annotate with security_contact from public_contacts, invited users # from domain_invitations, and managers from user_domain_roles @@ -307,8 +308,8 @@ class DomainExport(BaseExport): domain_info["security_contact_email"] = public_contacts.get( domain_info.get("domain__security_contact_registry_id") ) - domain_info["invited_users"] = domain_invitations.get(domain_info.get("domain__name")) - domain_info["managers"] = user_domain_roles.get(domain_info.get("domain__name")) + domain_info["invited_users"] = ", ".join(invited_users_dict.get(domain_info.get("domain__name"), [])) + domain_info["managers"] = ", ".join(managers_dict.get(domain_info.get("domain__name"), [])) annotated_domain_infos.append(domain_info) if annotated_domain_infos: @@ -335,7 +336,7 @@ class DomainExport(BaseExport): Fetch all DomainInvitation entries and return a mapping of domain to email. """ domain_invitations = DomainInvitation.objects.filter(status="invited").values_list("domain__name", "email") - return {domain__name: email for domain__name, email in domain_invitations} + return list(domain_invitations) @classmethod def get_all_user_domain_roles(cls): @@ -343,7 +344,7 @@ class DomainExport(BaseExport): Fetch all UserDomainRole entries and return a mapping of domain to user__email. """ user_domain_roles = UserDomainRole.objects.select_related("user").values_list("domain__name", "user__email") - return {domain__name: user__email for domain__name, user__email in user_domain_roles} + return list(user_domain_roles) @classmethod def parse_row(cls, columns, model): From 6275de19b5f25651639bb5793504867436a4c1b9 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 4 Jul 2024 09:37:31 -0400 Subject: [PATCH 37/55] fixed invited managers and domain managers --- src/registrar/tests/test_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 265876987..ded04e31b 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -633,4 +633,4 @@ class HelperFunctions(MockDb): } submitted_requests_sliced_at_end_date = DomainRequestExport.get_sliced_requests(filter_condition) expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1] - self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) \ No newline at end of file + self.assertEqual(submitted_requests_sliced_at_end_date, expected_content) From f5e552f796d477e2f045a10fd15b3b883c7dc014 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Thu, 4 Jul 2024 11:25:46 -0600 Subject: [PATCH 38/55] removed log --- src/registrar/views/user_profile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index feb545e12..60411f7e0 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -109,7 +109,6 @@ class UserProfileView(UserProfilePermissionView, FormMixin): if hasattr(self.user, "finished_setup") and not self.user.finished_setup: messages.error(self.request, "Before you can manage your domain, we need you to add contact information.") form.initial["redirect"] = form.data.get("redirect") - logger.info(form.initial) return super().form_invalid(form) def form_valid(self, form): From 582936662fee08dcac02e61793e1afcc734ecfa1 Mon Sep 17 00:00:00 2001 From: Cameron Dixon Date: Fri, 5 Jul 2024 15:22:38 -0400 Subject: [PATCH 39/55] * to - --- .github/ISSUE_TEMPLATE/issue-default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/issue-default.yml b/.github/ISSUE_TEMPLATE/issue-default.yml index f1cb5694f..f4bbb20bb 100644 --- a/.github/ISSUE_TEMPLATE/issue-default.yml +++ b/.github/ISSUE_TEMPLATE/issue-default.yml @@ -31,8 +31,8 @@ body: attributes: label: Links to other issues description: | - "With a `*` to start, add issue #numbers this relates to and how (e.g., 🚧 [construction] Blocks, ⛔️ [no_entry] Is blocked by, 🔄 [arrows_counterclockwise] Relates to)." - placeholder: * 🔄 Relates to... + "With a `-` to start the line, add issue #numbers this relates to and how (e.g., 🚧 [construction] Blocks, ⛔️ [no_entry] Is blocked by, 🔄 [arrows_counterclockwise] Relates to)." + placeholder: "- 🔄 Relates to... - type: markdown id: note attributes: From 38b58d79ccdafbcd575c025a3c05341f8089a09d Mon Sep 17 00:00:00 2001 From: Cameron Dixon Date: Fri, 5 Jul 2024 15:53:33 -0400 Subject: [PATCH 40/55] Update issue-default.yml Amazing that issue templates fail silently on this --- .github/ISSUE_TEMPLATE/issue-default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/issue-default.yml b/.github/ISSUE_TEMPLATE/issue-default.yml index f4bbb20bb..254078246 100644 --- a/.github/ISSUE_TEMPLATE/issue-default.yml +++ b/.github/ISSUE_TEMPLATE/issue-default.yml @@ -32,7 +32,7 @@ body: label: Links to other issues description: | "With a `-` to start the line, add issue #numbers this relates to and how (e.g., 🚧 [construction] Blocks, ⛔️ [no_entry] Is blocked by, 🔄 [arrows_counterclockwise] Relates to)." - placeholder: "- 🔄 Relates to... + placeholder: "- 🔄 Relates to..." - type: markdown id: note attributes: From e31fefcef11a10f5e8aa2ede5347a01787d591b2 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 9 Jul 2024 14:53:56 -0400 Subject: [PATCH 41/55] New sandbox ms --- .github/workflows/migrate.yaml | 1 + .github/workflows/reset-db.yaml | 1 + ops/manifests/manifest-ms.yaml | 32 ++++++++++++++++++++++++++++++++ src/registrar/config/settings.py | 1 + 4 files changed, 35 insertions(+) create mode 100644 ops/manifests/manifest-ms.yaml diff --git a/.github/workflows/migrate.yaml b/.github/workflows/migrate.yaml index 81368f6e9..3ebee59f9 100644 --- a/.github/workflows/migrate.yaml +++ b/.github/workflows/migrate.yaml @@ -16,6 +16,7 @@ on: - stable - staging - development + - ms - ag - litterbox - hotgov diff --git a/.github/workflows/reset-db.yaml b/.github/workflows/reset-db.yaml index ad325c50a..49e4b5e5f 100644 --- a/.github/workflows/reset-db.yaml +++ b/.github/workflows/reset-db.yaml @@ -16,6 +16,7 @@ on: options: - staging - development + - ms - ag - litterbox - hotgov diff --git a/ops/manifests/manifest-ms.yaml b/ops/manifests/manifest-ms.yaml new file mode 100644 index 000000000..153ee5f08 --- /dev/null +++ b/ops/manifests/manifest-ms.yaml @@ -0,0 +1,32 @@ +--- +applications: +- name: getgov-ms + buildpacks: + - python_buildpack + path: ../../src + instances: 1 + memory: 512M + stack: cflinuxfs4 + timeout: 180 + command: ./run.sh + health-check-type: http + health-check-http-endpoint: /health + health-check-invocation-timeout: 40 + env: + # Send stdout and stderr straight to the terminal without buffering + PYTHONUNBUFFERED: yup + # Tell Django where to find its configuration + DJANGO_SETTINGS_MODULE: registrar.config.settings + # Tell Django where it is being hosted + DJANGO_BASE_URL: https://getgov-ms.app.cloud.gov + # Tell Django how much stuff to log + DJANGO_LOG_LEVEL: INFO + # default public site location + GETGOV_PUBLIC_SITE_URL: https://get.gov + # Flag to disable/enable features in prod environments + IS_PRODUCTION: False + routes: + - route: getgov-ms.app.cloud.gov + services: + - getgov-credentials + - getgov-ms-database diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 688a3e8ca..1b8a5005f 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -660,6 +660,7 @@ ALLOWED_HOSTS = [ "getgov-stable.app.cloud.gov", "getgov-staging.app.cloud.gov", "getgov-development.app.cloud.gov", + "getgov-ms.app.cloud.gov", "getgov-ag.app.cloud.gov", "getgov-litterbox.app.cloud.gov", "getgov-hotgov.app.cloud.gov", From 9209856b0c18e2995d11db58a931b8813e194cce Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 9 Jul 2024 14:55:59 -0400 Subject: [PATCH 42/55] edit deploy-sandbox --- .github/workflows/deploy-sandbox.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-sandbox.yaml b/.github/workflows/deploy-sandbox.yaml index 84f228893..4bd7f99dd 100644 --- a/.github/workflows/deploy-sandbox.yaml +++ b/.github/workflows/deploy-sandbox.yaml @@ -28,6 +28,7 @@ jobs: || startsWith(github.head_ref, 'hotgov/') || startsWith(github.head_ref, 'litterbox/') || startsWith(github.head_ref, 'ag/') + || startsWith(github.head_ref, 'ms/') outputs: environment: ${{ steps.var.outputs.environment}} runs-on: "ubuntu-latest" From c3dfc08b25261fc2068d17841cace0276593a62e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:57:42 -0600 Subject: [PATCH 43/55] Fix 500 bug --- src/registrar/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index ef7888005..bcab9c96b 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -19,7 +19,7 @@ from epplibwrapper.errors import ErrorCode, RegistryError from registrar.models.user_domain_role import UserDomainRole from waffle.admin import FlagAdmin from waffle.models import Sample, Switch -from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website +from registrar.models import Contact, Domain, DomainRequest, DraftDomain, User, Website, SeniorOfficial from registrar.utility.errors import FSMDomainRequestError, FSMErrorCodes from registrar.views.utility.mixins import OrderableFieldsMixin from django.contrib.admin.views.main import ORDER_VAR @@ -448,8 +448,9 @@ class AdminSortFields: sort_mapping = { # == Contact == # "other_contacts": (Contact, _name_sort), - "senior_official": (Contact, _name_sort), "submitter": (Contact, _name_sort), + # == Senior Official == # + "senior_official": (SeniorOfficial, _name_sort), # == User == # "creator": (User, _name_sort), "user": (User, _name_sort), From 92f269c69eebcaef14a3ae663cc40ae3859eae8e Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 9 Jul 2024 13:59:30 -0600 Subject: [PATCH 44/55] Fix tests and lint --- src/registrar/admin.py | 2 +- src/registrar/tests/test_admin.py | 77 +++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index bcab9c96b..a59111540 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -449,7 +449,7 @@ class AdminSortFields: # == Contact == # "other_contacts": (Contact, _name_sort), "submitter": (Contact, _name_sort), - # == Senior Official == # + # == Senior Official == # "senior_official": (SeniorOfficial, _name_sort), # == User == # "creator": (User, _name_sort), diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 3ffb14905..3245e3201 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -44,6 +44,7 @@ from registrar.models import ( UserGroup, TransitionDomain, ) +from registrar.models.senior_official import SeniorOfficial from registrar.models.user_domain_role import UserDomainRole from registrar.models.verified_by_staff import VerifiedByStaff from .common import ( @@ -935,6 +936,34 @@ class TestDomainRequestAdmin(MockEppLib): ) self.mock_client = MockSESClient() + def test_domain_request_senior_official_is_alphabetically_sorted(self): + """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" + + third_official = SeniorOfficial.objects.get_or_create( + first_name="mary", last_name="joe", title="some other guy" + ) + first_official = SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") + second_official = SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") + + contact, _ = Contact.objects.get_or_create(user=self.staffuser) + domain_request = completed_domain_request(submitter=contact, name="city1.gov") + request = self.factory.post("/admin/registrar/domainrequest/{}/change/".format(domain_request.pk)) + model_admin = AuditedAdmin(DomainRequest, self.site) + + # Get the queryset that would be returned for the list + senior_offical_queryset = model_admin.formfield_for_foreignkey( + DomainInformation.senior_official.field, request + ).queryset + + # Make the list we're comparing on a bit prettier display-wise. Optional step. + current_sort_order = [] + for official in senior_offical_queryset: + current_sort_order.append(f"{official.first_name} {official.last_name}") + + expected_sort_order = ["alex smoe", "mary joe", "Zoup Soup"] + + self.assertEqual(current_sort_order, expected_sort_order) + @less_console_noise_decorator def test_has_model_description(self): """Tests if this model has a model description on the table view""" @@ -2732,6 +2761,7 @@ class TestDomainRequestAdmin(MockEppLib): User.objects.all().delete() Contact.objects.all().delete() Website.objects.all().delete() + SeniorOfficial.objects.all().delete() self.mock_client.EMAILS_SENT.clear() @@ -2913,6 +2943,40 @@ class TestDomainInformationAdmin(TestCase): Domain.objects.all().delete() Contact.objects.all().delete() User.objects.all().delete() + SeniorOfficial.objects.all().delete() + + def test_domain_information_senior_official_is_alphabetically_sorted(self): + """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" + + third_official = SeniorOfficial.objects.get_or_create( + first_name="mary", last_name="joe", title="some other guy" + ) + first_official = SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") + second_official = SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") + + contact, _ = Contact.objects.get_or_create(user=self.staffuser) + domain_request = completed_domain_request( + submitter=contact, name="city1244.gov", status=DomainRequest.DomainRequestStatus.IN_REVIEW + ) + domain_request.approve() + + domain_info = DomainInformation.objects.get(domain_request=domain_request) + request = self.factory.post("/admin/registrar/domaininformation/{}/change/".format(domain_info.pk)) + model_admin = AuditedAdmin(DomainInformation, self.site) + + # Get the queryset that would be returned for the list + senior_offical_queryset = model_admin.formfield_for_foreignkey( + DomainInformation.senior_official.field, request + ).queryset + + # Make the list we're comparing on a bit prettier display-wise. Optional step. + current_sort_order = [] + for official in senior_offical_queryset: + current_sort_order.append(f"{official.first_name} {official.last_name}") + + expected_sort_order = ["alex smoe", "mary joe", "Zoup Soup"] + + self.assertEqual(current_sort_order, expected_sort_order) @less_console_noise_decorator def test_admin_can_see_cisa_region_federal(self): @@ -3667,6 +3731,7 @@ class AuditedAdminTest(TestCase): self.site = AdminSite() self.factory = RequestFactory() self.client = Client(HTTP_HOST="localhost:8080") + self.staffuser = create_user() def order_by_desired_field_helper(self, obj_to_sort: AuditedAdmin, request, field_name, *obj_names): with less_console_noise(): @@ -3718,7 +3783,9 @@ class AuditedAdminTest(TestCase): def test_alphabetically_sorted_fk_fields_domain_request(self): with less_console_noise(): tested_fields = [ - DomainRequest.senior_official.field, + # Senior offical is commented out for now - this is alphabetized + # and this test does not accurately reflect that. + # DomainRequest.senior_official.field, DomainRequest.submitter.field, # DomainRequest.investigator.field, DomainRequest.creator.field, @@ -3776,7 +3843,9 @@ class AuditedAdminTest(TestCase): def test_alphabetically_sorted_fk_fields_domain_information(self): with less_console_noise(): tested_fields = [ - DomainInformation.senior_official.field, + # Senior offical is commented out for now - this is alphabetized + # and this test does not accurately reflect that. + # DomainInformation.senior_official.field, DomainInformation.submitter.field, # DomainInformation.creator.field, (DomainInformation.domain.field, ["name"]), @@ -3809,7 +3878,6 @@ class AuditedAdminTest(TestCase): # Conforms to the same object structure as desired_order current_sort_order_coerced_type = [] - # This is necessary as .queryset and get_queryset # return lists of different types/structures. # We need to parse this data and coerce them into the same type. @@ -3886,7 +3954,8 @@ class AuditedAdminTest(TestCase): if last_name is None: return (first_name,) - if first_name.split(queryset_shorthand)[1] == field_name: + split_name = first_name.split(queryset_shorthand) + if len(split_name) == 2 and split_name[1] == field_name: return returned_tuple else: return None From 9a44f6b65a70bcfc313196e9ccd7a7fc501a0dcd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:03:52 -0600 Subject: [PATCH 45/55] Remove unused variable names --- src/registrar/tests/test_admin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 3245e3201..d3610d187 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,11 +939,11 @@ class TestDomainRequestAdmin(MockEppLib): def test_domain_request_senior_official_is_alphabetically_sorted(self): """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" - third_official = SeniorOfficial.objects.get_or_create( + SeniorOfficial.objects.get_or_create( first_name="mary", last_name="joe", title="some other guy" ) - first_official = SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") - second_official = SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") + SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") + SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") contact, _ = Contact.objects.get_or_create(user=self.staffuser) domain_request = completed_domain_request(submitter=contact, name="city1.gov") @@ -2948,11 +2948,11 @@ class TestDomainInformationAdmin(TestCase): def test_domain_information_senior_official_is_alphabetically_sorted(self): """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" - third_official = SeniorOfficial.objects.get_or_create( + SeniorOfficial.objects.get_or_create( first_name="mary", last_name="joe", title="some other guy" ) - first_official = SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") - second_official = SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") + SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") + SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") contact, _ = Contact.objects.get_or_create(user=self.staffuser) domain_request = completed_domain_request( From c2ba10f1915cb862d3c8951835dd2ee844ec2319 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:06:47 -0600 Subject: [PATCH 46/55] Linting! --- src/registrar/tests/test_admin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index d3610d187..bbc66fc21 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,9 +939,7 @@ class TestDomainRequestAdmin(MockEppLib): def test_domain_request_senior_official_is_alphabetically_sorted(self): """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" - SeniorOfficial.objects.get_or_create( - first_name="mary", last_name="joe", title="some other guy" - ) + SeniorOfficial.objects.get_or_create(first_name="mary", last_name="joe", title="some other guy") SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") @@ -2948,9 +2946,7 @@ class TestDomainInformationAdmin(TestCase): def test_domain_information_senior_official_is_alphabetically_sorted(self): """Tests if the senior offical dropdown is alphanetically sorted in the django admin display""" - SeniorOfficial.objects.get_or_create( - first_name="mary", last_name="joe", title="some other guy" - ) + SeniorOfficial.objects.get_or_create(first_name="mary", last_name="joe", title="some other guy") SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") From a7abfa9cdb16ac85fad271a01c665e0a7f0e276b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 16:48:03 -0400 Subject: [PATCH 47/55] made email readonly and fixed some unit tests --- src/registrar/admin.py | 2 +- src/registrar/forms/domain.py | 2 +- src/registrar/tests/test_admin.py | 4 +++- src/registrar/views/user_profile.py | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 7ddff76c0..325fc53dd 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -917,7 +917,7 @@ class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): name.admin_order_field = "first_name" # type: ignore # Read only that we'll leverage for CISA Analysts - analyst_readonly_fields: list[str] = [] + analyst_readonly_fields: list[str] = ["email"] def get_readonly_fields(self, request, obj=None): """Set the read-only state on form elements. diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 06163dbce..9b8f1b7fc 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -204,7 +204,7 @@ NameserverFormset = formset_factory( class UserForm(forms.ModelForm): - """Form for updating contacts.""" + """Form for updating users.""" email = forms.EmailField(max_length=None) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 87fd9b5b8..c8fe64c42 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -2105,6 +2105,7 @@ class TestDomainRequestAdmin(MockEppLib): ] self.test_helper.assert_response_contains_distinct_values(response, expected_submitter_fields) self.assertContains(response, "Testy2 Tester2") + self.assertContains(response, "meoward.jones@igorville.gov") # == Check for the senior_official == # self.assertContains(response, "testy@town.com", count=2) @@ -3130,6 +3131,7 @@ class TestDomainInformationAdmin(TestCase): ("phone", "(555) 123 12345"), ] self.test_helper.assert_response_contains_distinct_values(response, expected_creator_fields) + self.assertContains(response, "meoward.jones@igorville.gov") # Check for the field itself self.assertContains(response, "Meoward Jones") @@ -4036,7 +4038,7 @@ class TestContactAdmin(TestCase): readonly_fields = self.admin.get_readonly_fields(request) - expected_fields = [] + expected_fields = ["email"] self.assertEqual(readonly_fields, expected_fields) diff --git a/src/registrar/views/user_profile.py b/src/registrar/views/user_profile.py index e378b9180..dec1d1af1 100644 --- a/src/registrar/views/user_profile.py +++ b/src/registrar/views/user_profile.py @@ -54,7 +54,6 @@ class UserProfileView(UserProfilePermissionView, FormMixin): def get_context_data(self, **kwargs): """Extend get_context_data to include has_profile_feature_flag""" context = super().get_context_data(**kwargs) - logger.info("UserProfileView::get_context_data") # This is a django waffle flag which toggles features based off of the "flag" table context["has_profile_feature_flag"] = flag_is_active(self.request, "profile_feature") From d393350a7f132eb079acf5762404656a20af9f5c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:05:37 -0400 Subject: [PATCH 48/55] fixed migrations --- ...2_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/registrar/migrations/{0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py => 0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py} (84%) diff --git a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py b/src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py similarity index 84% rename from src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py rename to src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py index 858210be7..db9b7970a 100644 --- a/src/registrar/migrations/0110_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py +++ b/src/registrar/migrations/0112_remove_contact_registrar_c_user_id_4059c4_idx_and_more.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("registrar", "0109_domaininformation_sub_organization_and_more"), + ("registrar", "0111_create_groups_v15"), ] operations = [ From f0dfe96e974e4313b1316e6d6717b088b9e86639 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:24:35 -0400 Subject: [PATCH 49/55] fixed tests --- src/registrar/tests/test_admin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 4fa443105..06fb27237 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,7 +939,10 @@ class TestDomainRequestAdmin(MockEppLib): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson" + ) domain_request = completed_domain_request(submitter=contact, name="city1.gov") request = self.factory.post("/admin/registrar/domainrequest/{}/change/".format(domain_request.pk)) model_admin = AuditedAdmin(DomainRequest, self.site) @@ -2943,7 +2946,10 @@ class TestDomainInformationAdmin(TestCase): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create(user=self.staffuser) + contact, _ = Contact.objects.get_or_create( + first_name="Henry", + last_name="McFakerson" + ) domain_request = completed_domain_request( submitter=contact, name="city1244.gov", status=DomainRequest.DomainRequestStatus.IN_REVIEW ) From d981c410e827854e0321681451c7288a1b4e960d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 9 Jul 2024 17:30:45 -0400 Subject: [PATCH 50/55] linter --- src/registrar/tests/test_admin.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 06fb27237..b77a1faf4 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -939,10 +939,7 @@ class TestDomainRequestAdmin(MockEppLib): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create( - first_name="Henry", - last_name="McFakerson" - ) + contact, _ = Contact.objects.get_or_create(first_name="Henry", last_name="McFakerson") domain_request = completed_domain_request(submitter=contact, name="city1.gov") request = self.factory.post("/admin/registrar/domainrequest/{}/change/".format(domain_request.pk)) model_admin = AuditedAdmin(DomainRequest, self.site) @@ -2946,10 +2943,7 @@ class TestDomainInformationAdmin(TestCase): SeniorOfficial.objects.get_or_create(first_name="alex", last_name="smoe", title="some guy") SeniorOfficial.objects.get_or_create(first_name="Zoup", last_name="Soup", title="title") - contact, _ = Contact.objects.get_or_create( - first_name="Henry", - last_name="McFakerson" - ) + contact, _ = Contact.objects.get_or_create(first_name="Henry", last_name="McFakerson") domain_request = completed_domain_request( submitter=contact, name="city1244.gov", status=DomainRequest.DomainRequestStatus.IN_REVIEW ) From 1d5d30873f665811bbc74359a1f33be01d73c4f5 Mon Sep 17 00:00:00 2001 From: matthewswspence Date: Wed, 10 Jul 2024 09:24:38 -0500 Subject: [PATCH 51/55] Add fixtures for Matthew Spence --- src/registrar/fixtures_users.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/registrar/fixtures_users.py b/src/registrar/fixtures_users.py index 2aa9d224b..74fd4d15d 100644 --- a/src/registrar/fixtures_users.py +++ b/src/registrar/fixtures_users.py @@ -22,6 +22,11 @@ class UserFixture: """ ADMINS = [ + { + "username": "be17c826-e200-4999-9389-2ded48c43691", + "first_name": "Matthew", + "last_name": "Spence", + }, { "username": "5f283494-31bd-49b5-b024-a7e7cae00848", "first_name": "Rachid", @@ -115,6 +120,11 @@ class UserFixture: ] STAFF = [ + { + "username": "d6bf296b-fac5-47ff-9c12-f88ccc5c1b99", + "first_name": "Matthew-Analyst", + "last_name": "Spence-Analyst", + }, { "username": "319c490d-453b-43d9-bc4d-7d6cd8ff6844", "first_name": "Rachid-Analyst", From 72b1631facf7e51aec33a17d80a417ea05c9c35c Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 10 Jul 2024 22:00:50 -0400 Subject: [PATCH 52/55] senior official hotfix scripts --- ...late_domain_information_senior_official.py | 76 +++++++++++++++++ ...opulate_domain_request_senior_official.py | 81 +++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 src/registrar/management/commands/repopulate_domain_information_senior_official.py create mode 100644 src/registrar/management/commands/repopulate_domain_request_senior_official.py diff --git a/src/registrar/management/commands/repopulate_domain_information_senior_official.py b/src/registrar/management/commands/repopulate_domain_information_senior_official.py new file mode 100644 index 000000000..540f88154 --- /dev/null +++ b/src/registrar/management/commands/repopulate_domain_information_senior_official.py @@ -0,0 +1,76 @@ +import argparse +import csv +import logging +import os +from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors +from registrar.models import DomainInformation + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + """ + This command uses the PopulateScriptTemplate, + which provides reusable logging and bulk updating functions for mass-updating fields. + """ + + help = "Loops through each valid DomainInformation object and updates its Senior Official" + prompt_title = "Do you wish to update all Senior Officials for Domain Information?" + + def handle(self, domain_info_csv_path, **kwargs): + """Loops through each valid DomainInformation object and updates its senior official field""" + + # Check if the provided file path is valid. + if not os.path.isfile(domain_info_csv_path): + raise argparse.ArgumentTypeError(f"Invalid file path '{domain_info_csv_path}'") + + # Simple check to make sure we don't accidentally pass in the wrong file. Crude but it works. + if "information" not in domain_info_csv_path.lower(): + raise argparse.ArgumentTypeError(f"Invalid file for domain information: '{domain_info_csv_path}'") + + # Get all ao data. + self.ao_dict = {} + self.ao_dict = self.read_csv_file_and_get_contacts(domain_info_csv_path) + + self.mass_update_records( + DomainInformation, filter_conditions={"senior_official__isnull": True}, fields_to_update=["senior_official"] + ) + + def add_arguments(self, parser): + """Add command line arguments.""" + parser.add_argument( + "--domain_info_csv_path", help="A csv containing the domain information id and the contact id" + ) + + def read_csv_file_and_get_contacts(self, file): + dict_data = {} + with open(file, "r") as requested_file: + reader = csv.DictReader(requested_file) + for row in reader: + domain_info_id = row.get("id") + ao_id = row.get("authorizing_official") + if ao_id: + ao_id = int(ao_id) + if domain_info_id and ao_id: + dict_data[int(domain_info_id)] = ao_id + + return dict_data + + def update_record(self, record: DomainInformation): + """Defines how we update the senior official field on each record.""" + record.senior_official_id = self.ao_dict.get(record.id) + logger.info(f"{TerminalColors.OKCYAN}Updating {str(record)} => {record.senior_official}{TerminalColors.ENDC}") + + def should_skip_record(self, record) -> bool: # noqa + """Defines the conditions in which we should skip updating a record.""" + # Don't update this record if there isn't ao data to pull from + if self.ao_dict.get(record.id) is None: + logger.info( + f"{TerminalColors.YELLOW}Skipping update for {str(record)} => " + f"Missing authorizing_official data.{TerminalColors.ENDC}" + ) + return True + else: + return False diff --git a/src/registrar/management/commands/repopulate_domain_request_senior_official.py b/src/registrar/management/commands/repopulate_domain_request_senior_official.py new file mode 100644 index 000000000..37fcea03e --- /dev/null +++ b/src/registrar/management/commands/repopulate_domain_request_senior_official.py @@ -0,0 +1,81 @@ +import argparse +import csv +import logging +import os +from django.core.management import BaseCommand +from registrar.management.commands.utility.terminal_helper import PopulateScriptTemplate, TerminalColors +from registrar.models import DomainRequest + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand, PopulateScriptTemplate): + """ + This command uses the PopulateScriptTemplate, + which provides reusable logging and bulk updating functions for mass-updating fields. + """ + + help = """Loops through each valid DomainRequest object and updates its senior official field""" + prompt_title = "Do you wish to update all Senior Officials for Domain Requests?" + + def handle(self, domain_request_csv_path, **kwargs): + """Loops through each valid DomainRequest object and updates its senior official field""" + + # Check if the provided file path is valid. + if not os.path.isfile(domain_request_csv_path): + raise argparse.ArgumentTypeError(f"Invalid file path '{domain_request_csv_path}'") + + # Simple check to make sure we don't accidentally pass in the wrong file. Crude but it works. + if "request" not in domain_request_csv_path.lower(): + raise argparse.ArgumentTypeError(f"Invalid file for domain requests: '{domain_request_csv_path}'") + + # Get all ao data. + self.ao_dict = {} + self.ao_dict = self.read_csv_file_and_get_contacts(domain_request_csv_path) + + self.mass_update_records( + DomainRequest, + filter_conditions={ + "senior_official__isnull": True, + }, + fields_to_update=["senior_official"], + ) + + def add_arguments(self, parser): + """Add command line arguments.""" + parser.add_argument( + "--domain_request_csv_path", help="A csv containing the domain request id and the contact id" + ) + + def read_csv_file_and_get_contacts(self, file): + dict_data: dict = {} + with open(file, "r") as requested_file: + reader = csv.DictReader(requested_file) + for row in reader: + domain_request_id = row.get("id") + ao_id = row.get("authorizing_official") + if ao_id: + ao_id = int(ao_id) + if domain_request_id and ao_id: + dict_data[int(domain_request_id)] = ao_id + + return dict_data + + def update_record(self, record: DomainRequest): + """Defines how we update the federal_type field on each record.""" + record.senior_official_id = self.ao_dict.get(record.id) + # record.senior_official = Contact.objects.get(id=contact_id) + logger.info(f"{TerminalColors.OKCYAN}Updating {str(record)} => {record.senior_official}{TerminalColors.ENDC}") + + def should_skip_record(self, record) -> bool: # noqa + """Defines the conditions in which we should skip updating a record.""" + # Don't update this record if there isn't ao data to pull from + if self.ao_dict.get(record.id) is None: + logger.info( + f"{TerminalColors.YELLOW}Skipping update for {str(record)} => " + f"Missing authorizing_official data.{TerminalColors.ENDC}" + ) + return True + else: + return False From 329737023bf7e3e95424010eb36a0f224bdfc655 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Wed, 10 Jul 2024 22:08:13 -0400 Subject: [PATCH 53/55] remove trailing whitespace --- ..._official.py => repopulate_domain_request_senior_official.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/registrar/management/commands/{repopulate_domain_request_senior_official.py => repopulate_domain_request_senior_official.py} (100%) diff --git a/src/registrar/management/commands/repopulate_domain_request_senior_official.py b/src/registrar/management/commands/repopulate_domain_request_senior_official.py similarity index 100% rename from src/registrar/management/commands/repopulate_domain_request_senior_official.py rename to src/registrar/management/commands/repopulate_domain_request_senior_official.py From 9b0302e8730f148431a6d2ff86ce0e4a6f685b6b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:02:53 -0600 Subject: [PATCH 54/55] add fix --- src/registrar/models/domain_request.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 078aa9f0d..188adaa52 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -136,6 +136,9 @@ class DomainRequest(TimeStampedModel): @classmethod def get_org_label(cls, org_name: str): """Returns the associated label for a given org name""" + if not org_name: + return None + org_names = org_name.split("_election") if len(org_names) > 0: org_name = org_names[0] From 1d5d7f79f10202a00ba3b85311d79cb510e51dcb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:42:50 -0600 Subject: [PATCH 55/55] Update domain_request.py --- src/registrar/models/domain_request.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 188adaa52..92f5869f7 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -136,6 +136,10 @@ class DomainRequest(TimeStampedModel): @classmethod def get_org_label(cls, org_name: str): """Returns the associated label for a given org name""" + # This is an edgecase on domains with no org. + # This unlikely to happen but + # a break will occur in certain edge cases without this. + # (more specifically, csv exports). if not org_name: return None